feat(xlsx): pictures and chart placeholders on sheets

docs/features.md

@@ -15,7 +15,7 @@
 | Format | Supported                                                                                                                                                                                                      | Not yet                                                                    |
 |--------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------|
 | DOCX   | inherited styles, run formatting, lists/numbering, tables (merge, border, shading, table style), images, hyperlinks, header/footer, PAGE field, text boxes, page/section breaks, text columns, tracked changes | footnote body, charts, exact float wrapping, paragraphs split across pages |
-| XLSX   | shared strings, number/date formats, font/fill/border/align, merge, freeze, hidden columns/rows, wrap, formula calculation (135 functions, recalculated on edit), multiple sheets                              | charts, conditional formatting, images                                     |
+| XLSX   | shared strings, number/date formats, font/fill/border/align, merge, freeze, hidden columns/rows, wrap, formula calculation (135 functions, recalculated on edit), multiple sheets, images (over the cells)     | charts (placeholder), shapes, conditional formatting                       |
 | PPTX   | master/layout/theme, placeholder inheritance, ~50 preset shapes, custGeom, images (crop), tables + table style, groups, bullet/autonum, autofit                                                                | charts (placeholder), SmartArt, animations, video playback                 |
 
 Password-protected (encrypted) files are not supported.

src/core/worker/engine.ts

@@ -7,6 +7,7 @@ import { BlobSource, openSource, type DocSource } from '../source'
 import { decodeText, extOf, HAS_SOURCE, mayBePlain, plainFormat, plainToOffice, ungzipped } from '../plain'
 import { XlsxWorkbook } from '../xlsx/workbook'
 import { XlsxEditor } from '../xlsx/editor'
+import { readSheetDrawings } from '../xlsx/drawings'
 import { loadDelimited } from '../xlsx/import'
 import type { BookOp, CellInput, FindOptions, FmtPatch, Range, SheetOp } from '../xlsx/types'
 import type { PParaSpec, ShapeOp, SlideOp } from '../pptx/types'
@@ -207,6 +208,11 @@ export class Engine {
         return this.need(this.xlsx).block(args[0] as number, args[1] as number, args[2] as number)
       case 'xlsx.cell':
         return this.need(this.xlsx).cell(args[0] as number, args[1] as number, args[2] as number)
+      case 'xlsx.drawings': {
+        const wb = this.need(this.xlsx)
+        const s = wb.sheets[args[0] as number]
+        return s ? readSheetDrawings(wb.pkg, s.path) : []
+      }
       case 'xlsx.find':
         return this.need(this.xlsx).find(args[0] as number, args[1] as string, args[2] as FindOptions)
       case 'xlsx.text':

src/core/worker/protocol.ts

@@ -7,6 +7,7 @@ import type { POp } from '../../collab/pptx-ops'
 import type { PptxCollabResult } from '../../collab/pptx-worker'
 import type { CanonStyle, DocxMeta, Block, EditOp, EditResult, HFSet, HFSetResult, PageSet, PageSetResult, ParaSpec, StyleInfo, StyleLook, StyleSetResult, TableOp } from '../docx/types'
 import type { BlockOut, BookOp, CellDetail, CellInput, FindMatch, FindOptions, FmtPatch, Range, SheetOp, SheetProgress, WorkbookMeta, XlsxEditResult } from '../xlsx/types'
+import type { SheetDrawing } from '../xlsx/drawings'
 import type { PParaSpec, PresentationMeta, ShapeOp, SlideModel, SlideOp } from '../pptx/types'
 
 export interface OpenParams {
@@ -69,6 +70,8 @@ export interface Methods {
   'xlsx.load'(sheet: number): void
   'xlsx.block'(sheet: number, r0: number, r1: number): BlockOut
   'xlsx.cell'(sheet: number, r: number, c: number): CellDetail | undefined
+  /** the sheet's pictures and charts, where they are anchored */
+  'xlsx.drawings'(sheet: number): SheetDrawing[]
   /** cells whose shown value (or formula) contains the text */
   'xlsx.find'(sheet: number, query: string, opts?: FindOptions): FindMatch[]
   /** the sheet as delimited text (the values as shown): `sep` a comma (with a byte order mark) or a tab */

src/core/xlsx/drawings.ts

@@ -0,0 +1,111 @@
+/**
+ * The pictures and charts on a sheet, read from its drawing part (`xl/drawings/drawingN.xml`, linked from the sheet's
+ * rels), each with the place it is anchored to: two cell corners (`twoCellAnchor`), one corner and a size
+ * (`oneCellAnchor`), or a position on the sheet and a size (`absoluteAnchor`). A picture carries its media part, which
+ * the view shows where the browser can draw it (not EMF / WMF); a chart carries its kind and title, for a placeholder
+ * the size of the chart. Shapes, text boxes and the like are left out, as are drawings in a group the anchor's own
+ * frame does not hold.
+ */
+import { isWebImage, type OpcPackage } from '../opc'
+import { desc, descAll, kid, num, textOf, type XEl } from '../xml'
+
+/** A cell corner: 0-based column and row, and the offset into that cell in CSS pixels */
+export interface DrawingCorner {
+  c: number
+  r: number
+  dx: number
+  dy: number
+}
+
+export interface SheetDrawing {
+  kind: 'picture' | 'chart'
+  from?: DrawingCorner
+  to?: DrawingCorner
+  /** position from the sheet's top left, px (absoluteAnchor) */
+  at?: { x: number; y: number }
+  /** px (oneCellAnchor, absoluteAnchor) */
+  size?: { w: number; h: number }
+  /** a picture's media part, where the browser can draw it */
+  target?: string
+  /** a chart's title, or a picture's description */
+  title: string
+  /** a chart's kind: 'column', 'bar', 'line', 'pie', … ('' where it is none the reader knows) */
+  chart?: string
+}
+
+/** DrawingML measures in EMU: 914400 to the inch, 96 CSS pixels to the inch */
+const EMU_PER_PX = 9525
+
+const CHART_KINDS: Record<string, string> = {
+  barChart: 'bar', bar3DChart: 'bar', lineChart: 'line', line3DChart: 'line', pieChart: 'pie', pie3DChart: 'pie', ofPieChart: 'pie',
+  doughnutChart: 'doughnut', areaChart: 'area', area3DChart: 'area', scatterChart: 'scatter', bubbleChart: 'bubble', radarChart: 'radar',
+  stockChart: 'stock', surfaceChart: 'surface', surface3DChart: 'surface',
+}
+
+/** an attribute whatever its namespace prefix: `r:id` is the usual, not the only one */
+const attr = (el: XEl | undefined, local: string) => {
+  if (!el) return undefined
+  for (const [k, v] of Object.entries(el.a)) if (k === local || k.endsWith(':' + local)) return v
+  return undefined
+}
+
+const corner = (el: XEl | undefined): DrawingCorner | undefined =>
+  el
+    ? {
+        c: num(textOf(kid(el, 'col')), 0)!,
+        r: num(textOf(kid(el, 'row')), 0)!,
+        dx: num(textOf(kid(el, 'colOff')), 0)! / EMU_PER_PX,
+        dy: num(textOf(kid(el, 'rowOff')), 0)! / EMU_PER_PX,
+      }
+    : undefined
+
+/** a chart's kind and title, from its chart part */
+async function chartOf(pkg: OpcPackage, part: string): Promise<{ chart: string; title: string }> {
+  const root = await pkg.xml(part)
+  const plot = desc(root, 'plotArea')
+  const type = plot?.c.find((e) => e.n.endsWith('Chart'))
+  let chart = type ? (CHART_KINDS[type.n] ?? '') : ''
+  // a bar chart is drawn with upright columns unless its direction says bars
+  if (type?.n.startsWith('bar') && attr(kid(type, 'barDir'), 'val') !== 'bar') chart = 'column'
+  const title = desc(root, 'title')
+  return { chart, title: title ? descAll(title, 't').map((t) => t.t).join('').trim() : '' }
+}
+
+/** The pictures and charts on the sheet at `sheetPath`, in the order they are drawn */
+export async function readSheetDrawings(pkg: OpcPackage, sheetPath: string): Promise<SheetDrawing[]> {
+  const sheetRels = await pkg.rels(sheetPath)
+  const out: SheetDrawing[] = []
+  for (const rel of sheetRels.values()) {
+    if (rel.external || !rel.type.endsWith('/drawing') || !pkg.has(rel.target)) continue
+    const root = await pkg.xml(rel.target)
+    const rels = await pkg.rels(rel.target)
+    for (const anchor of root?.c ?? []) {
+      if (anchor.n !== 'twoCellAnchor' && anchor.n !== 'oneCellAnchor' && anchor.n !== 'absoluteAnchor') continue
+      const ext = kid(anchor, 'ext')
+      const pos = kid(anchor, 'pos')
+      const place = {
+        from: corner(kid(anchor, 'from')),
+        to: corner(kid(anchor, 'to')),
+        at: pos ? { x: num(pos.a.x, 0)! / EMU_PER_PX, y: num(pos.a.y, 0)! / EMU_PER_PX } : undefined,
+        size: ext ? { w: num(ext.a.cx, 0)! / EMU_PER_PX, h: num(ext.a.cy, 0)! / EMU_PER_PX } : undefined,
+      }
+      const chartRef = desc(anchor, 'chart')
+      const blip = desc(anchor, 'blip')
+      if (chartRef) {
+        const target = rels.get(attr(chartRef, 'id') ?? '')
+        const chart = target && !target.external && pkg.has(target.target) ? await chartOf(pkg, target.target) : { chart: '', title: '' }
+        out.push({ kind: 'chart', ...place, ...chart })
+      } else if (blip) {
+        const target = rels.get(attr(blip, 'embed') ?? '')
+        const pr = desc(anchor, 'cNvPr')
+        out.push({
+          kind: 'picture',
+          ...place,
+          target: target && !target.external && isWebImage(target.target) ? target.target : undefined,
+          title: pr?.a.descr || pr?.a.title || '',
+        })
+      }
+    }
+  }
+  return out
+}

src/index.ts

@@ -16,6 +16,7 @@ export type { DocSource } from './core/source'
 export type { DocKind } from './core/opc'
 export type * from './core/docx/types'
 export type * from './core/xlsx/types'
+export type { DrawingCorner, SheetDrawing } from './core/xlsx/drawings'
 export type * from './core/pptx/types'
 
 export { startCollab, CollabHub, CollabSession, MemoryTransport, WebSocketTransport, type Collab, type CollabOptions, type CollabStatus, type Transport } from './collab'

src/react/edit/Toolbar.tsx

@@ -42,6 +42,7 @@ const P: Record<string, ReactNode> = {
   view: <path d="M2 12s3.5-7 10-7 10 7 10 7-3.5 7-10 7S2 12 2 12m10 3a3 3 0 1 0 0-6 3 3 0 0 0 0 6" />,
   close: <path d="M6 6l12 12M18 6 6 18" />,
   image: <path d="M4 5h16v14H4zM4 16l5-5 4 4 3-3 4 4M15.5 9.5h.01" />,
+  chart: <path d="M4 20h16M7 20v-8M12 20V5M17 20v-5" />,
   table: <path d="M4 5h16v14H4zM4 10h16M4 15h16M10 5v14M15 5v14" />,
   link: <path d="M10 14a4 4 0 0 0 5.7 0l3-3a4 4 0 0 0-5.7-5.7l-1 1M14 10a4 4 0 0 0-5.7 0l-3 3a4 4 0 0 0 5.7 5.7l1-1" />,
   headerFooter: <path d="M6 3h12v18H6zM8 6h8M8 18h8M12 16.5v3" strokeDasharray="0" />,

src/react/styles.css

@@ -397,6 +397,15 @@
   position: absolute;
   overflow: hidden;
 }
+/* XLSX: pictures and charts over the cells; clicks go through to the cell below */
+.de-xl-pic { position: absolute; z-index: 2; object-fit: fill; pointer-events: none; user-select: none; }
+.de-xl-drawing {
+  position: absolute; z-index: 2; box-sizing: border-box; display: flex; flex-direction: column; align-items: center; justify-content: center;
+  gap: 3px; padding: 6px; overflow: hidden; border: 1px dashed var(--de-border); border-radius: 4px; background: color-mix(in srgb, var(--de-chrome) 94%, transparent);
+  color: var(--de-muted); font: 12px/1.3 system-ui, sans-serif; text-align: center; pointer-events: none;
+}
+.de-xl-drawing svg { color: var(--de-accent); }
+.de-xl-drawing strong { color: var(--de-text); font-weight: 600; }
 .de-gl-h,
 .de-gl-v {
   position: absolute;

src/react/xlsx/XlsxView.tsx

@@ -15,6 +15,7 @@ import { Toast } from '../common/Dialog'
 import { FindBar } from '../docx/FindBar'
 import { XlsxToolbar, type MergeMode } from './XlsxToolbar'
 import { Axis } from './axis'
+import { drawingBox, drawingEl, drawingExtent, useSheetDrawings } from './drawings'
 import { fitNumber, fontPx } from './fit'
 import { sheetPrintable } from './print'
 import { useCommentsCtl, useCommentsState, type CommentsCtl, type PlacedThread } from '../comments/Comments'
@@ -556,11 +557,14 @@ function SheetGrid({ store, sheet, meta, client, edit, setEdit, hold, onSheet }:
   const [resizing, setResizing] = useState<{ axis: 'col' | 'row'; i: number; start: number; size: number } | null>(null)
 
   const { rowAxis, colAxis, frozen } = s
+  const drawings = useSheetDrawings(client, sheet)
+  // the sheet reaches as far as its drawings, past its last filled cell
+  const drawn = drawingExtent(drawings, colAxis, rowAxis)
   const HW = Math.max(40, 12 + 7 * String(Math.max(s.rows, 1000)).length)
   const viewW = Math.max(0, size.w - HW)
   const viewH = Math.max(0, size.h - HH)
-  rowAxis.count = Math.max(s.rows + (s.done ? 20 : 1), Math.ceil(viewH / rowAxis.def) + 1, frozen.rows + 1, sel.fr + 2)
-  colAxis.count = Math.max(s.cols + 3, 26, Math.ceil(viewW / Math.max(1, colAxis.def)) + 1, frozen.cols + 1, sel.fc + 2)
+  rowAxis.count = Math.max(s.rows + (s.done ? 20 : 1), Math.ceil(viewH / rowAxis.def) + 1, frozen.rows + 1, sel.fr + 2, drawn.rows + 1)
+  colAxis.count = Math.max(s.cols + 3, 26, Math.ceil(viewW / Math.max(1, colAxis.def)) + 1, frozen.cols + 1, sel.fc + 2, drawn.cols + 1)
   const frozenW = colAxis.pos(frozen.cols)
   const frozenH = rowAxis.pos(frozen.rows)
   const totalW = colAxis.total()
@@ -723,6 +727,16 @@ function SheetGrid({ store, sheet, meta, client, edit, setEdit, hold, onSheet }:
         out.push(cellEl(r, cell, undefined, undefined, filled))
       }
     }
+    // pictures and charts over the cells, where they reach into this pane
+    const px0 = colAxis.pos(cc0)
+    const px1 = colAxis.pos(cc1 + 1)
+    const py0 = rowAxis.pos(rr0)
+    const py1 = rowAxis.pos(rr1 + 1)
+    drawings.forEach((d, i) => {
+      const b = drawingBox(d, colAxis, rowAxis)
+      if (!b || b.x > px1 || b.x + b.w < px0 || b.y > py1 || b.y + b.h < py0) return
+      out.push(drawingEl('dr' + i, d, { left: b.x - ox, top: b.y - oy, width: b.w, height: b.h }))
+    })
     // selection range + active cell
     if (rect.r1 >= rr0 && rect.r0 <= rr1 && rect.c1 >= cc0 && rect.c0 <= cc1 && !oneCell) {
       out.push(

src/react/xlsx/drawings.tsx

@@ -0,0 +1,81 @@
+import { useEffect, useState, type CSSProperties, type ReactNode } from 'react'
+import type { DocumentClient } from '../../core/worker/client'
+import type { SheetDrawing } from '../../core/xlsx/drawings'
+import { Icon } from '../edit/Toolbar'
+import type { Axis } from './axis'
+
+/** a sheet's drawing with its picture's object URL, where the browser can draw it */
+export interface ShownDrawing extends SheetDrawing {
+  url?: string
+}
+
+/** The pictures and charts of the sheet (`xlsx.drawings`), read again when another sheet is shown */
+export function useSheetDrawings(client: DocumentClient, sheet: number): ShownDrawing[] {
+  const [list, setList] = useState<ShownDrawing[]>([])
+  useEffect(() => {
+    let live = true
+    setList([])
+    client
+      .call('xlsx.drawings', sheet)
+      .then((drawings) => Promise.all(drawings.map(async (d) => ({ ...d, url: d.target ? await client.mediaUrl(d.target).catch(() => undefined) : undefined }))))
+      .then((shown) => live && setList(shown))
+      .catch((e: Error) => console.warn('[office-engine] drawings not read:', e.message))
+    return () => {
+      live = false
+    }
+  }, [client, sheet])
+  return list
+}
+
+/**
+ * Where a drawing lies on the sheet, px from its top left: anchored to cells it moves and grows with their rows and
+ * columns (a hidden one is 0 wide); given by a size it keeps that size.
+ */
+export function drawingBox(d: SheetDrawing, cols: Axis, rows: Axis): { x: number; y: number; w: number; h: number } | null {
+  let x: number
+  let y: number
+  if (d.from) {
+    x = cols.pos(d.from.c) + d.from.dx
+    y = rows.pos(d.from.r) + d.from.dy
+  } else if (d.at) {
+    x = d.at.x
+    y = d.at.y
+  } else return null
+  const w = d.to ? cols.pos(d.to.c) + d.to.dx - x : (d.size?.w ?? 0)
+  const h = d.to ? rows.pos(d.to.r) + d.to.dy - y : (d.size?.h ?? 0)
+  return w >= 2 && h >= 2 ? { x, y, w, h } : null
+}
+
+/** How many rows and columns the sheet needs to reach every drawing (a drawing given by a size taken over default-sized cells) */
+export function drawingExtent(list: SheetDrawing[], cols: Axis, rows: Axis): { rows: number; cols: number } {
+  let r = 0
+  let c = 0
+  for (const d of list) {
+    const end = d.to ?? (d.from && d.size ? { c: d.from.c + Math.ceil(d.size.w / cols.def), r: d.from.r + Math.ceil(d.size.h / rows.def) } : undefined)
+    const fromAt = !end && d.at && d.size ? { c: Math.ceil((d.at.x + d.size.w) / cols.def), r: Math.ceil((d.at.y + d.size.h) / rows.def) } : undefined
+    const e = end ?? fromAt
+    if (e) {
+      r = Math.max(r, e.r + 1)
+      c = Math.max(c, e.c + 1)
+    }
+  }
+  return { rows: r, cols: c }
+}
+
+const CHART_NAMES: Record<string, string> = {
+  column: 'Column chart', bar: 'Bar chart', line: 'Line chart', pie: 'Pie chart', doughnut: 'Doughnut chart', area: 'Area chart',
+  scatter: 'Scatter chart', bubble: 'Bubble chart', radar: 'Radar chart', stock: 'Stock chart', surface: 'Surface chart',
+}
+
+/** A drawing as shown: the picture itself, or a box of its size saying what is there (a chart, a picture the browser cannot draw) */
+export function drawingEl(key: string, d: ShownDrawing, style: CSSProperties): ReactNode {
+  if (d.url) return <img key={key} className="de-xl-pic" src={d.url} alt={d.title} style={style} draggable={false} />
+  const what = d.kind === 'chart' ? CHART_NAMES[d.chart ?? ''] ?? 'Chart' : 'Picture'
+  return (
+    <div key={key} className="de-xl-drawing" style={style} title={[what, d.title].filter(Boolean).join(': ')}>
+      <Icon name={d.kind === 'chart' ? 'chart' : 'image'} size={24} />
+      <strong>{what}</strong>
+      {d.title && <span>{d.title}</span>}
+    </div>
+  )
+}