Decode HEIC chat attachments to JPEG via the OS decoder

src/desktop/ipc-bridge.ts

@@ -32,6 +32,7 @@ import { log } from '../server/log';
  */
 const LOCAL_IPC_ARGS: Record<string, ArgSpec[]> = {
   'files:preview': [a.string],
+  'files:previewData': [a.string, a.optional(a.nullish(a.string)), a.optional(a.nullish(a.string))],
   'files:download': [a.string],
   'cfolders:reveal': [a.string],
   'client:pair': [a.string, a.string],

src/desktop/local/index.ts

@@ -2,7 +2,7 @@ import { app, dialog, shell, type BrowserWindow } from 'electron';
 import { join } from 'node:path';
 import { handleLocal } from '../ipc-bridge';
 import { ensureFilesRoot } from '../../server/files/store';
-import { imagePreviewDataUrl } from '../../server/pi/attachments';
+import { imagePreviewDataUrl, imagePreviewFromBytes } from '../../server/pi/attachments';
 import { connectedFolderPath } from '../../server/workspace/connected-folders';
 import { workspaceRoot } from '../../server/workspace/paths';
 import { exportState } from '../../server/workspace/state-transfer';
@@ -262,10 +262,16 @@ export function registerLocalIpc(deps: LocalIpcDeps): void {
     revealable('Your Files folder');
     await shell.openPath(await ensureFilesRoot());
   });
-  // Read-only, and reached only from the `att.path` branch of
-  // renderer/attachments.ts — i.e. for an image the user picked or dropped, which
-  // by construction is on the client's own disk.
+  // Read-only, and reached from renderer/attachments.ts: a path for images the
+  // user picked or dropped (on this disk), or pasted bytes for HEIC that
+  // Chromium cannot paint until the OS decoder turns them into JPEG.
   handleLocal('files:preview', (_e, path: string) => imagePreviewDataUrl(path));
+  // Pasted HEIC has no on-disk path; the renderer sends the bytes so Chromium
+  // can show a JPEG thumbnail (it cannot paint image/heic).
+  handleLocal(
+    'files:previewData',
+    (_e, dataBase64: string, mime?: string, name?: string) => imagePreviewFromBytes(dataBase64, mime, name)
+  );
 
   /**
    * Fetch one file out of the server's Files folder and put it where downloads

src/desktop/proxy.ts

@@ -19,6 +19,7 @@ import {
   type StartTurnInput,
   type TurnAttachment
 } from '../shared/types';
+import { convertHeicAttachments } from '../server/pi/heic';
 import { uploadFile } from './file-transfer';
 import { createOfflineCache } from './offline-cache';
 import type { OAuthCourier } from './oauth-courier';
@@ -59,10 +60,12 @@ import { updateClientQuickChat, withClientSettings } from './settings';
 //                                                about the build installed HERE
 //                                                (see desktop/updates.ts)
 //   dialog:openFiles, dialog:openDirectory       native pickers
-//   files:reveal, files:preview                  shell.showItemInFolder; preview
-//                                                reads an image path that, by
-//                                                construction, is on the client's
-//                                                own disk (the `att.path` branch
+//   files:reveal, files:preview, files:previewData
+//                                                shell.showItemInFolder; preview
+//                                                reads an image path (or pasted
+//                                                bytes) that, by construction, is
+//                                                on the client's own disk (the
+//                                                `att.path` / dataBase64 branches
 //                                                of renderer/attachments.ts)
 //   files:download                               GET /files/<rel>, saved into this
 //                                                machine's Downloads folder and
@@ -112,16 +115,18 @@ import { updateClientQuickChat, withClientSettings } from './settings';
 //                              push stream is one this machine asked for — the
 //                              stream is a broadcast, and every other device
 //                              paired to the same server sees it too.
-//   backend:startTurn,         both carry paths to files on THIS disk, which is
-//   files:add                  only a thing the server can read when it is on
+//   backend:startTurn,         HEIC is decoded to JPEG here first (macOS sips /
+//   files:add                  a PATH tool) so a remote Linux server never has
+//                              to. Remaining paths to files on THIS disk are
+//                              only a thing the server can read when it is on
 //                              this disk too. When it isn't, the bytes are
 //                              streamed up first and the paths are replaced with
-//                              handles to them — see attachmentsForServer(). The
-//                              REMOTE case only: a local install keeps handing
-//                              over paths, because copying every pasted
-//                              screenshot through loopback to prove a point
-//                              would be a cost with nothing on the other side
-//                              of it.
+//                              handles to them — see attachmentsForTurn(). The
+//                              REMOTE case only for the upload half: a local
+//                              install keeps handing over paths, because copying
+//                              every pasted screenshot through loopback to prove
+//                              a point would be a cost with nothing on the other
+//                              side of it.
 //
 // SERVER-OWNED — everything else (~110 channels). The server's registry IS the
 // surface; this client asks for it at connect time (GET /channels) rather than
@@ -346,34 +351,33 @@ export function createServerProxy(deps: ProxyDeps): ServerProxy {
   const signInStarted: WrappedChannel = { before: () => deps.oauthCourier.expectSignIn() };
 
   /**
-   * Replace every on-disk path in a set of attachments with a handle to bytes the
-   * server now has. Pasted images (`dataBase64`) already travel in the envelope
-   * and are left exactly as they are — they are small, they are already on the
-   * wire, and uploading them separately would be strictly more work.
+   * Decode HEIC to JPEG on THIS machine (sips / PATH tool) so a Mac client
+   * talking to a Linux server still works — the server never has to ship HEVC.
+   * Then, when remote, replace remaining on-disk paths with upload handles.
    *
-   * A failure here is deliberately fatal to the call. The alternative is sending
-   * the message with the attachment quietly missing, which reads to the user as
-   * the assistant ignoring the thing they attached; throwing instead leaves the
-   * message in the composer, with the reason on screen, ready to send again.
+   * Pasted images (`dataBase64`) already travel in the envelope. After a HEIC
+   * conversion they are JPEG `dataBase64` and are left exactly as they are.
+   *
+   * A failure to *upload* is deliberately fatal to the call. The alternative is
+   * sending the message with the attachment quietly missing, which reads to the
+   * user as the assistant ignoring the thing they attached; throwing instead
+   * leaves the message in the composer, with the reason on screen, ready to send
+   * again. A HEIC that will not decode is left unchanged so the server can skip
+   * it with a named note rather than failing the whole send.
    */
-  async function attachmentsForServer(atts: TurnAttachment[]): Promise<TurnAttachment[]> {
+  async function attachmentsForTurn(atts: TurnAttachment[]): Promise<TurnAttachment[]> {
+    const converted = await convertHeicAttachments(atts);
+    if (!deps.remote) return converted;
     return Promise.all(
-      atts.map(async (att) => {
+      converted.map(async (att) => {
         if (!att.path) return att;
         return { ...att, path: await uploadFile({ url: base, token: deps.token }, att.path) };
       })
     );
   }
 
-  /** The remote half of `backend:startTurn` and `files:add`; absent when local. */
+  /** Path rewrite for Files-panel drops; turn attachments go through attachmentsForTurn. */
   const uploadPaths: Record<string, WrappedChannel> = {
-    'backend:startTurn': {
-      before: async ([input]) => {
-        const turn = input as StartTurnInput;
-        if (!turn?.attachments?.length) return;
-        return [{ ...turn, attachments: await attachmentsForServer(turn.attachments) }];
-      }
-    },
     'files:add': {
       before: async ([paths, subdir]) => {
         const list = paths as string[];
@@ -384,12 +388,21 @@ export function createServerProxy(deps: ProxyDeps): ServerProxy {
     }
   };
 
+  const startTurnWrap: WrappedChannel = {
+    before: async ([input]) => {
+      const turn = input as StartTurnInput;
+      if (!turn?.attachments?.length) return;
+      return [{ ...turn, attachments: await attachmentsForTurn(turn.attachments) }];
+    }
+  };
+
   const wrapped: Readonly<Record<string, WrappedChannel>> = {
     'chats:open': {
       before: ([threadId]) => deps.threadOpened(threadId as string)
     },
     'auth:providerLogin': signInStarted,
     'mcp:login': signInStarted,
+    'backend:startTurn': startTurnWrap,
     ...(deps.remote ? uploadPaths : {}),
     ...Object.fromEntries(SETTINGS_CHANNELS.map((c) => [c, mergeSettingsAnswer])),
     'settings:updateQuickChat': {

src/preload/index.ts

@@ -118,6 +118,8 @@ const api: StemApi = {
   revealFiles: () => ipcRenderer.invoke('files:reveal'),
   downloadFile: (rel: string) => ipcRenderer.invoke('files:download', rel),
   previewImage: (path: string) => ipcRenderer.invoke('files:preview', path),
+  previewImageData: (dataBase64: string, mime?: string, name?: string) =>
+    ipcRenderer.invoke('files:previewData', dataBase64, mime, name),
 
   listConnectedFolders: () => ipcRenderer.invoke('cfolders:list'),
   addConnectedFolders: (paths: string[]) => ipcRenderer.invoke('cfolders:add', paths),

src/renderer/attachments.ts

@@ -6,7 +6,23 @@
 
 import type { ChatMessage, MessageAttachment, TurnAttachment } from '../shared/types';
 
-const IMAGE_EXT = /\.(png|jpe?g|gif|webp)$/i;
+const IMAGE_EXT = /\.(png|jpe?g|gif|webp|heic|heif|hif)$/i;
+
+function isHeic(att: TurnAttachment): boolean {
+  const mime = att.mime?.toLowerCase() ?? '';
+  if (mime === 'image/jpeg' || mime === 'image/png' || mime === 'image/gif' || mime === 'image/webp') {
+    return false;
+  }
+  if (
+    mime === 'image/heic' ||
+    mime === 'image/heif' ||
+    mime === 'image/heic-sequence' ||
+    mime === 'image/heif-sequence'
+  ) {
+    return true;
+  }
+  return /\.(heic|heif|hif)$/i.test(att.name || att.path || '');
+}
 
 function isImage(att: TurnAttachment): boolean {
   if (att.mime?.toLowerCase().startsWith('image/')) return true;
@@ -17,10 +33,12 @@ function isImage(att: TurnAttachment): boolean {
  * Synchronous attachment shape for the optimistic bubble. On-disk images need an
  * IPC read before their thumbnail is available, so they begin as ordinary chips
  * and are upgraded by `toMessageAttachments()` without delaying the send itself.
+ * Pasted HEIC is the same: Chromium cannot paint `image/heic`, so it starts as a
+ * chip until the main process has decoded it to JPEG.
  */
 export function optimisticMessageAttachments(atts: TurnAttachment[]): MessageAttachment[] {
   return atts.map((att) => {
-    if (isImage(att) && att.dataBase64) {
+    if (isImage(att) && att.dataBase64 && !isHeic(att)) {
       const mime = att.mime || 'image/png';
       return { kind: 'image', name: att.name, mime, dataUrl: `data:${mime};base64,${att.dataBase64}` };
     }
@@ -32,13 +50,21 @@ export async function toMessageAttachments(atts: TurnAttachment[]): Promise<Mess
   return Promise.all(
     atts.map(async (att): Promise<MessageAttachment> => {
       if (isImage(att)) {
-        const mime = att.mime || 'image/png';
         if (att.dataBase64) {
-          return { kind: 'image', name: att.name, mime, dataUrl: `data:${mime};base64,${att.dataBase64}` };
+          if (isHeic(att)) {
+            const dataUrl = await window.stem.previewImageData(att.dataBase64, att.mime, att.name);
+            if (dataUrl) return { kind: 'image', name: att.name, mime: 'image/jpeg', dataUrl };
+          } else {
+            const mime = att.mime || 'image/png';
+            return { kind: 'image', name: att.name, mime, dataUrl: `data:${mime};base64,${att.dataBase64}` };
+          }
         }
         if (att.path) {
           const dataUrl = await window.stem.previewImage(att.path);
-          if (dataUrl) return { kind: 'image', name: att.name, mime, dataUrl };
+          if (dataUrl) {
+            const mime = isHeic(att) ? 'image/jpeg' : att.mime || 'image/png';
+            return { kind: 'image', name: att.name, mime, dataUrl };
+          }
         }
       }
       return { kind: 'file', name: att.name };

src/server/pi/attachments.ts

@@ -2,9 +2,10 @@
 //
 // pi's `prompt` RPC accepts images natively (`images: [{type:'image', data, mimeType}]`).
 // It has no slot for arbitrary files, so text-like files are inlined into the message as
-// fenced blocks, PDFs are inlined as their extracted text layer, and other binary files
-// are rejected. This module is the single place that reads attachment bytes (from
-// `dataBase64` or an on-disk `path`) and classifies them.
+// fenced blocks, PDFs are inlined as their extracted text layer, HEIC/HEIF is decoded
+// to JPEG via the OS (see heic.ts), and other binary files are rejected. This module
+// is the single place that reads attachment bytes (from `dataBase64` or an on-disk
+// `path`) and classifies them.
 //
 // `path` is the CLIENT's path, which is only a path we can read when the client is on
 // this machine. A client whose server is elsewhere streams the bytes to POST /upload
@@ -15,6 +16,7 @@ import { readFile } from 'node:fs/promises';
 import { extname } from 'node:path';
 import { isUploadHandle, resolveUploadHandle } from '../files/staging';
 import { extractPdfText } from '../folder-index/pdf';
+import { heicToJpeg, isHeicAttachment, isHeicNameOrMime } from './heic';
 import type { TurnAttachment } from '../../shared/types';
 
 /** pi `ImageContent` — the shape of each entry in the prompt's `images` array. */
@@ -29,7 +31,7 @@ export interface ResolvedAttachments {
   images: PiImageContent[];
   /** Fenced file contents appended to the message text. */
   textBlocks: string[];
-  /** Basenames of attachments skipped: unsupported binaries, unreadable bytes, or PDFs with no text layer. */
+  /** Basenames of attachments skipped: unsupported binaries, unreadable bytes, PDFs with no text layer, or HEIC this machine cannot decode. */
   rejected: string[];
 }
 
@@ -71,13 +73,26 @@ function imageMimeFor(att: TurnAttachment, ext: string): string | null {
   const mime = att.mime?.toLowerCase();
   if (mime?.startsWith('image/')) {
     // Normalise to a type pi accepts; drop unknown image subtypes to the ext map.
+    // HEIC is not in this list — Chromium and most model APIs cannot take it, so
+    // it is decoded to JPEG first (see maybeHeicJpeg).
     if (mime === 'image/png' || mime === 'image/jpeg' || mime === 'image/gif' || mime === 'image/webp') {
       return mime;
     }
   }
   return IMAGE_EXT_MIME[ext] ?? null;
 }
 
+/** JPEG base64 for a HEIC attachment, or null if this machine cannot decode it. */
+async function maybeHeicJpeg(
+  att: TurnAttachment,
+  bytes: Buffer
+): Promise<{ data: string; mimeType: 'image/jpeg' } | null> {
+  if (!isHeicAttachment(att, bytes)) return null;
+  const jpeg = await heicToJpeg(bytes);
+  if (!jpeg) return null;
+  return { data: jpeg.toString('base64'), mimeType: 'image/jpeg' };
+}
+
 function looksTextual(att: TurnAttachment, ext: string, bytes: Buffer): boolean {
   if (att.mime?.toLowerCase().startsWith('text/')) return true;
   if (TEXT_EXT.has(ext)) return true;
@@ -110,6 +125,31 @@ function fenceText(name: string, lang: string, body: string, truncated: boolean)
   return `Attached file: ${name}\n\`\`\`${lang}\n${body}${note}\n\`\`\``;
 }
 
+/**
+ * Turn image bytes into a Chromium-displayable `data:` URL. HEIC is decoded to
+ * JPEG first — Electron cannot paint `image/heic`. Null if it isn't a supported
+ * image or this machine cannot decode it.
+ */
+export async function imagePreviewFromBytes(
+  dataBase64: string,
+  mime?: string,
+  name?: string
+): Promise<string | null> {
+  try {
+    const bytes = Buffer.from(dataBase64, 'base64');
+    const jpeg = await maybeHeicJpeg({ name: name || 'image', mime, dataBase64 }, bytes);
+    if (jpeg) return `data:${jpeg.mimeType};base64,${jpeg.data}`;
+    const ext = extname(name || '').toLowerCase();
+    const resolved = imageMimeFor({ name: name || 'image', mime }, ext);
+    if (!resolved) return null;
+    return `data:${resolved};base64,${dataBase64}`;
+  } catch {
+    // quiet: this is the thumbnail in the live bubble, not the attachment — the
+    // send path reads the same bytes again, and that read is the one that reports.
+    return null;
+  }
+}
+
 /**
  * Read an on-disk image and return a `data:` URL for an inline thumbnail, or null if the
  * file isn't a supported image or can't be read. Used to preview dialog/drop-picked
@@ -118,9 +158,16 @@ function fenceText(name: string, lang: string, body: string, truncated: boolean)
 export async function imagePreviewDataUrl(path: string): Promise<string | null> {
   const ext = extname(path).toLowerCase();
   const mime = IMAGE_EXT_MIME[ext];
-  if (!mime) return null;
+  const heic = isHeicNameOrMime({ name: path, path });
+  if (!mime && !heic) return null;
   try {
     const bytes = await readFile(path);
+    if (heic || isHeicAttachment({ name: path, path }, bytes)) {
+      const jpeg = await heicToJpeg(bytes);
+      if (!jpeg) return null;
+      return `data:image/jpeg;base64,${jpeg.toString('base64')}`;
+    }
+    if (!mime) return null;
     return `data:${mime};base64,${bytes.toString('base64')}`;
   } catch {
     // quiet: this is the thumbnail in the live bubble, not the attachment — the
@@ -137,7 +184,8 @@ export async function resolveAttachments(atts: TurnAttachment[]): Promise<Resolv
     const imageMime = imageMimeFor(att, ext);
 
     // Fast path for pasted images: bytes are already base64, no decode round-trip needed.
-    if (imageMime && att.dataBase64 && !att.path) {
+    // HEIC never takes this path — it has to be decoded to JPEG first.
+    if (imageMime && att.dataBase64 && !att.path && !isHeicNameOrMime(att)) {
       out.images.push({ type: 'image', data: att.dataBase64, mimeType: imageMime });
       continue;
     }
@@ -148,6 +196,16 @@ export async function resolveAttachments(atts: TurnAttachment[]): Promise<Resolv
       continue;
     }
 
+    const fromHeic = await maybeHeicJpeg(att, bytes);
+    if (fromHeic) {
+      out.images.push({ type: 'image', data: fromHeic.data, mimeType: fromHeic.mimeType });
+      continue;
+    }
+    if (isHeicAttachment(att, bytes)) {
+      out.rejected.push(`${att.name} (could not decode HEIC)`);
+      continue;
+    }
+
     if (imageMime) {
       out.images.push({ type: 'image', data: bytes.toString('base64'), mimeType: imageMime });
       continue;