diff --git a/CHANGELOG.md b/CHANGELOG.md index 5e5b4ae..86ee95e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,17 @@ +## v1.3.0 (July 28th, 2026) + +- Added opt-in incremental (append-semantics) element data delivery: + - `subscribeToIncrementalElementData` on the client, which advertises + incremental delivery to the host and invokes its callback with + `WorkbookElementDataChunk` envelopes (`data`, `offset`, `isComplete`, + `totalRows`). + - `useIncrementalElementData` React hook, which accumulates chunks + internally and is a drop-in replacement for `usePaginatedElementData`. + - Hosts that do not support incremental delivery are unaffected: their + cumulative payloads are transparently delivered as replace-everything + chunks at offset 0, so plugins using the new API work against both host + behaviors. All existing APIs are unchanged. + ## v1.0.0 (September 23rd, 2022) `@sigmacomputing/plugin` has moved to https://github.com/sigmacomputing/plugin and diff --git a/package.json b/package.json index 4dcdf85..c601e76 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@sigmacomputing/plugin-sdk-root", - "version": "1.2.0", + "version": "1.3.0", "private": true, "description": "Sigma Computing Plugin Client SDK", "license": "MIT", diff --git a/packages/plugin-sdk/README.md b/packages/plugin-sdk/README.md index 7b0a0d2..76ab7ec 100644 --- a/packages/plugin-sdk/README.md +++ b/packages/plugin-sdk/README.md @@ -795,6 +795,70 @@ interface WorkbookElementData { } ``` +#### useIncrementalElementData() + +Drop-in replacement for `usePaginatedElementData()` that opts in to +incremental (append-semantics) data delivery. When the host supports it, each +page is delivered as a chunk containing only the new rows, so loading a large +element costs each row once instead of re-delivering the entire accumulated +data set on every page. When the host does not support incremental delivery, +the hook transparently falls back to today's cumulative behavior — no +branching code is required in the plugin. + +```ts +function useIncrementalElementData( + configId: string, +): [WorkbookElementData, () => void, IncrementalElementDataInfo]; +``` + +Arguments + +- `configId : string` - A workbook element’s unique identifier from the plugin config. + +Returns the accumulated row data from the specified element, a callback for +fetching more data, and progress metadata: + +```ts +interface IncrementalElementDataInfo { + rowCount: number; // rows accumulated so far + isComplete: boolean; // true once the host reports no more rows + totalRows?: number; // total rows in the source element, if the host reports it +} +``` + +> **Warning:** on hosts without incremental support, `isComplete` stays +> `false` forever — completion is a signal only incremental-capable hosts can +> send. Never drive an auto-load loop or a "load more" affordance from +> `isComplete` alone; use `rowCount` to detect whether a fetch actually made +> progress (if it stops growing, there is no more data). + +Example + +```ts +const [data, loadMore, { rowCount, isComplete }] = + useIncrementalElementData('source'); +``` + +Framework Agnostic Usage + +```ts +const unsubscribe = client.elements.subscribeToIncrementalElementData( + 'source', + chunk => { + // chunk.data contains only this chunk's rows; chunk.offset is the + // absolute row offset to apply them at. Hosts without incremental + // support deliver their cumulative payloads as replace-everything + // chunks at offset 0. + applyRowsAtOffset(chunk.data, chunk.offset); + }, +); +``` + +Use one subscription style per element: the delivery mode belongs to the +(plugin, element) subscription, so mixing `subscribeToElementData` and +`subscribeToIncrementalElementData` (or their hooks) on the same config +element is unsupported. + #### useVariable() Returns a given variable's value and a setter to update that variable diff --git a/packages/plugin-sdk/package.json b/packages/plugin-sdk/package.json index 36830b2..a5f3771 100644 --- a/packages/plugin-sdk/package.json +++ b/packages/plugin-sdk/package.json @@ -1,6 +1,6 @@ { "name": "@sigmacomputing/plugin", - "version": "1.2.0", + "version": "1.3.0", "description": "Sigma Computing Plugin Client SDK", "license": "MIT", "type": "module", diff --git a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts index aa7ff0f..ad4035e 100644 --- a/packages/plugin-sdk/src/client/__tests__/initialize.test.ts +++ b/packages/plugin-sdk/src/client/__tests__/initialize.test.ts @@ -638,6 +638,88 @@ describe('initialize', () => { expect(callback).not.toHaveBeenCalled(); }); + it('subscribeToIncrementalElementData subscribes with the incremental capability, dispatches chunks, and unsubscribes', () => { + const callback = vi.fn(); + const unsub = client.elements.subscribeToIncrementalElementData( + 'el1', + callback, + ); + + const sub = findPostMessage( + postMessageSpy, + 'wb:plugin:element:subscribe:data', + ); + expect(sub?.data.args).toEqual(['el1', { mode: 'incremental' }]); + + const chunk = { + data: { c1: [1, 2, 3] }, + offset: 0, + isComplete: false, + totalRows: 6, + }; + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: chunk, + error: null, + }); + expect(callback).toHaveBeenCalledWith(chunk); + + postMessageSpy.mockClear(); + callback.mockClear(); + unsub(); + const unsubMsg = findPostMessage( + postMessageSpy, + 'wb:plugin:element:unsubscribe:data', + ); + expect(unsubMsg?.data.args).toEqual(['el1']); + + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: chunk, + error: null, + }); + expect(callback).not.toHaveBeenCalled(); + }); + + it('subscribeToIncrementalElementData normalizes legacy cumulative payloads into replace chunks at offset 0', () => { + const callback = vi.fn(); + client.elements.subscribeToIncrementalElementData('el1', callback); + + const legacyData = { c1: [1, 2, 3], c2: ['a', 'b', 'c'] }; + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: legacyData, + error: null, + }); + expect(callback).toHaveBeenCalledWith({ + data: legacyData, + offset: 0, + isComplete: false, + }); + }); + + it('subscribeToIncrementalElementData treats envelopes with malformed offsets as legacy payloads', () => { + const callback = vi.fn(); + client.elements.subscribeToIncrementalElementData('el1', callback); + + for (const offset of [-1, 1.5, Number.NaN]) { + callback.mockClear(); + const malformed = { data: { c1: [1] }, offset, isComplete: true }; + sendWindowMessage({ + type: 'wb:plugin:element:el1:data', + result: malformed, + error: null, + }); + // Not recognized as a chunk: falls back to replace-at-0 normalization + // instead of corrupting chunk assembly downstream. + expect(callback).toHaveBeenCalledWith({ + data: malformed, + offset: 0, + isComplete: false, + }); + } + }); + it('fetchMoreElementData posts wb:plugin:element:fetch-more', () => { client.elements.fetchMoreElementData('el1'); const msg = findPostMessage( diff --git a/packages/plugin-sdk/src/client/initialize.ts b/packages/plugin-sdk/src/client/initialize.ts index 8f9c9fe..73fb902 100644 --- a/packages/plugin-sdk/src/client/initialize.ts +++ b/packages/plugin-sdk/src/client/initialize.ts @@ -4,11 +4,33 @@ import { PluginMessageResponse, PluginStyle, UrlParameter, + WorkbookElementData, + WorkbookElementDataChunk, WorkbookSelection, WorkbookVariable, } from '../types'; import { validateConfigId } from '../utils/error'; +// Every value in a legacy cumulative WorkbookElementData payload is a column +// array, so typed non-array `offset`/`isComplete`/`data` fields can only come +// from the incremental chunk envelope. Offsets must be non-negative integers; +// a payload with a malformed offset is treated as legacy data rather than +// letting a NaN/negative/fractional value corrupt chunk assembly downstream. +function isElementDataChunk( + result: WorkbookElementData | WorkbookElementDataChunk, +): result is WorkbookElementDataChunk { + const chunk = result as Partial; + return ( + result != null && + Number.isInteger(chunk.offset) && + (chunk.offset as number) >= 0 && + typeof chunk.isComplete === 'boolean' && + typeof chunk.data === 'object' && + chunk.data !== null && + !Array.isArray(chunk.data) + ); +} + export function initialize(): PluginInstance { const pluginConfig: Partial> = { config: {} as T, @@ -255,6 +277,33 @@ export function initialize(): PluginInstance { void execPromise('wb:plugin:element:unsubscribe:data', configId); }; }, + subscribeToIncrementalElementData(configId, callback) { + validateConfigId(configId, 'element'); + const eventName = `wb:plugin:element:${configId}:data`; + const onData = ( + result: WorkbookElementData | WorkbookElementDataChunk, + ) => { + if (isElementDataChunk(result)) { + callback(result); + } else { + // A host without incremental support ignores the subscribe + // options and keeps sending cumulative payloads. Deliver those as + // replace-everything chunks so consumers behave identically + // against either host. Legacy hosts never signal completion, so + // isComplete stays false. + callback({ data: result, offset: 0, isComplete: false }); + } + }; + on(eventName, onData); + void execPromise('wb:plugin:element:subscribe:data', configId, { + mode: 'incremental', + }); + + return () => { + off(eventName, onData); + void execPromise('wb:plugin:element:unsubscribe:data', configId); + }; + }, fetchMoreElementData(configId) { validateConfigId(configId, 'element'); void execPromise('wb:plugin:element:fetch-more', configId); diff --git a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx index 99f4cf7..ccdc0e5 100644 --- a/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx +++ b/packages/plugin-sdk/src/react/__tests__/hooks.test.tsx @@ -11,6 +11,7 @@ import { useEditorPanelConfig, useElementColumns, useElementData, + useIncrementalElementData, useInteraction, useLoadingState, usePaginatedElementData, @@ -272,6 +273,240 @@ describe('react/hooks', () => { }); }); + describe('useIncrementalElementData', () => { + it('subscribes to incremental data and concatenates chunks by offset', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + expect(sub.spy).toHaveBeenCalledWith('el1', expect.any(Function)); + expect(result.current[0]).toEqual({}); + expect(result.current[2]).toEqual({ rowCount: 0, isComplete: false }); + + act(() => + sub.emit({ + data: { c1: [1, 2], c2: ['a', 'b'] }, + offset: 0, + isComplete: false, + totalRows: 4, + }), + ); + act(() => + sub.emit({ + data: { c1: [3, 4], c2: ['c', 'd'] }, + offset: 2, + isComplete: true, + }), + ); + + expect(result.current[0]).toEqual({ + c1: [1, 2, 3, 4], + c2: ['a', 'b', 'c', 'd'], + }); + expect(result.current[2]).toEqual({ + rowCount: 4, + isComplete: true, + totalRows: 4, + }); + }); + + it('applies overlapping chunks idempotently by trusting the offset', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ data: { c1: [1, 2, 3] }, offset: 0, isComplete: false }), + ); + const overlapping = { + data: { c1: [3, 4] }, + offset: 2, + isComplete: false, + }; + act(() => sub.emit(overlapping)); + act(() => sub.emit(overlapping)); + + expect(result.current[0]).toEqual({ c1: [1, 2, 3, 4] }); + expect(result.current[2].rowCount).toBe(4); + }); + + it('preserves accumulated data when a terminal chunk is empty or omits a column', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ + data: { c1: [1, 2], c2: ['a', 'b'] }, + offset: 0, + isComplete: false, + }), + ); + // A chunk omitting c2 must not delete c2's accumulated rows. + act(() => + sub.emit({ data: { c1: [3, 4] }, offset: 2, isComplete: false }), + ); + // An empty terminal chunk only flips isComplete. + act(() => sub.emit({ data: {}, offset: 4, isComplete: true })); + + expect(result.current[0]).toEqual({ c1: [1, 2, 3, 4], c2: ['a', 'b'] }); + expect(result.current[2]).toEqual({ rowCount: 4, isComplete: true }); + }); + + it('replaces state wholesale and re-baselines totalRows on an offset-0 restart', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ + data: { c1: [1, 2, 3] }, + offset: 0, + isComplete: true, + totalRows: 3, + }), + ); + // Host refresh: new column set, no totalRows reported. + act(() => + sub.emit({ data: { c9: ['x'] }, offset: 0, isComplete: false }), + ); + + expect(result.current[0]).toEqual({ c9: ['x'] }); + expect(result.current[2]).toEqual({ rowCount: 1, isComplete: false }); + }); + + it('keeps rows at their absolute offsets for gaps and columns appearing mid-stream', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + act(() => + sub.emit({ data: { c1: [1, 2] }, offset: 0, isComplete: false }), + ); + // c2 first appears at offset 2; its rows must not land at index 0. + act(() => + sub.emit({ + data: { c1: [3, 4], c2: ['c', 'd'] }, + offset: 2, + isComplete: false, + }), + ); + + expect(result.current[0].c1).toEqual([1, 2, 3, 4]); + expect(result.current[0].c2.length).toBe(4); + expect(result.current[0].c2[2]).toBe('c'); + expect(result.current[0].c2[3]).toBe('d'); + expect(result.current[0].c2[0]).toBeUndefined(); + }); + + it('tolerates column ids that collide with Object.prototype members', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + // JSON.parse creates '__proto__' as an own enumerable property, which + // is exactly what a (hostile or buggy) wire payload can carry. + const data = JSON.parse( + '{"constructor": [1, 2], "toString": [3, 4], "__proto__": [5, 6]}', + ); + act(() => sub.emit({ data, offset: 0, isComplete: true })); + + expect(result.current[0]['constructor']).toEqual([1, 2]); + expect(result.current[0]['toString']).toEqual([3, 4]); + // '__proto__' is skipped rather than reparenting the accumulator. + expect(Object.getPrototypeOf(result.current[0])).toBe(Object.prototype); + expect(result.current[2].rowCount).toBe(2); + }); + + it('matches legacy cumulative payloads exactly when the host lacks incremental support', () => { + // Exercise the real client end-to-end: the host ignores the capability + // option and re-sends the entire accumulated data set on every page. + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + + const sendLegacyData = (data: Record) => { + window.dispatchEvent( + new MessageEvent('message', { + data: { + type: 'wb:plugin:element:el1:data', + result: data, + error: null, + }, + }), + ); + }; + + act(() => sendLegacyData({ c1: [1, 2, 3] })); + act(() => sendLegacyData({ c1: [1, 2, 3, 4, 5, 6] })); + + expect(result.current[0]).toEqual({ c1: [1, 2, 3, 4, 5, 6] }); + expect(result.current[2]).toEqual({ rowCount: 6, isComplete: false }); + }); + + it('returns a loadMore callback that fetches more data', () => { + stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const fetchSpy = vi.spyOn(client.elements, 'fetchMoreElementData'); + const { result } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + act(() => result.current[1]()); + expect(fetchSpy).toHaveBeenCalledWith('el1'); + }); + + it('does not subscribe and loadMore is a no-op when configId is falsy', () => { + const subSpy = vi.spyOn( + client.elements, + 'subscribeToIncrementalElementData', + ); + const fetchSpy = vi.spyOn(client.elements, 'fetchMoreElementData'); + const { result } = renderHook(() => useIncrementalElementData(''), { + wrapper: withProvider(client), + }); + expect(subSpy).not.toHaveBeenCalled(); + act(() => result.current[1]()); + expect(fetchSpy).not.toHaveBeenCalled(); + }); + + it('unsubscribes on unmount', () => { + const sub = stubSubscription( + client.elements, + 'subscribeToIncrementalElementData', + ); + const { unmount } = renderHook(() => useIncrementalElementData('el1'), { + wrapper: withProvider(client), + }); + unmount(); + expect(sub.unsubscribe).toHaveBeenCalled(); + }); + }); + describe('useConfig', () => { it('returns the full config when no key is provided', () => { vi.spyOn(client.config, 'get').mockReturnValue({ a: 1 }); diff --git a/packages/plugin-sdk/src/react/hooks.ts b/packages/plugin-sdk/src/react/hooks.ts index f3b9700..ccbe030 100644 --- a/packages/plugin-sdk/src/react/hooks.ts +++ b/packages/plugin-sdk/src/react/hooks.ts @@ -6,6 +6,7 @@ import { CustomPluginConfigOptions, WorkbookElementColumns, WorkbookElementData, + WorkbookElementDataChunk, WorkbookSelection, WorkbookVariable, PluginStyle, @@ -130,6 +131,117 @@ export function usePaginatedElementData( return [data, loadMore]; } +/** + * Progress metadata for incrementally accumulated element data + * @typedef {object} IncrementalElementDataInfo + * @property {number} rowCount Number of rows accumulated so far + * @property {boolean} isComplete True once the host reports no more rows are available + * @property {(number | undefined)} totalRows Total rows in the source element, if the host reports it + */ +export interface IncrementalElementDataInfo { + rowCount: number; + isComplete: boolean; + totalRows?: number; +} + +interface IncrementalElementDataState { + data: WorkbookElementData; + info: IncrementalElementDataInfo; +} + +const INITIAL_INCREMENTAL_STATE: IncrementalElementDataState = { + data: {}, + info: { rowCount: 0, isComplete: false }, +}; + +// Applies a chunk by trusting its absolute offset: rows before the offset are +// kept and rows at or after it are overwritten, so re-sent or overlapping +// chunks apply idempotently. An offset-0 chunk replaces the accumulated state +// wholesale (host refresh, or a cumulative payload from a host without +// incremental support, normalized upstream); a chunk at offset > 0 starts +// from the accumulated columns, so a chunk that omits a column — or an empty +// terminal chunk that only flips isComplete — cannot drop received rows. +function applyElementDataChunk( + prev: IncrementalElementDataState, + chunk: WorkbookElementDataChunk, +): IncrementalElementDataState { + const data: WorkbookElementData = chunk.offset === 0 ? {} : { ...prev.data }; + for (const colId of Object.keys(chunk.data)) { + // '__proto__' is never a real column id; assigning it would swap the + // object's prototype instead of adding a column. + if (colId === '__proto__') continue; + // Own-property check so inherited members (e.g. a column named + // 'constructor') can never be mistaken for accumulated rows. + const prevRows = Object.prototype.hasOwnProperty.call(data, colId) + ? data[colId] + : []; + const head = prevRows.slice(0, chunk.offset); + // Pad so rows always land at their absolute offset, even for a column + // first appearing mid-stream or a host that skips ahead. + head.length = chunk.offset; + data[colId] = head.concat(chunk.data[colId]); + } + const rowCount = Object.values(data).reduce( + (max, rows) => Math.max(max, rows.length), + 0, + ); + return { + data, + info: { + rowCount, + isComplete: chunk.isComplete, + // An offset-0 restart re-baselines the total instead of carrying a + // stale value from the previous load. + totalRows: + chunk.totalRows ?? + (chunk.offset === 0 ? undefined : prev.info.totalRows), + }, + }; +} + +/** + * Provides the data values from the corresponding config element, accumulated + * from incremental chunks, with a callback to fetch more in chunks of 25_000 + * data points. Drop-in replacement for usePaginatedElementData that avoids + * re-delivering already received rows when the host supports incremental + * delivery, and behaves identically to usePaginatedElementData when it does + * not. IMPORTANT: hosts without incremental support never signal completion, + * so isComplete stays false forever there — never drive an auto-load loop or + * a "load more" affordance from isComplete alone; use rowCount to detect + * whether a fetch actually made progress. + * @param {string} configId ID from the config for fetching incremental + * element data, with type: 'element' + * @returns {[WorkbookElementData, Function, IncrementalElementDataInfo]} + * Accumulated Element Data for the config element, a callback to fetch more + * data, and progress metadata + */ +export function useIncrementalElementData( + configId: string, +): [WorkbookElementData, () => void, IncrementalElementDataInfo] { + const client = usePlugin(); + const [state, setState] = React.useState( + INITIAL_INCREMENTAL_STATE, + ); + + const loadMore = React.useCallback(() => { + if (configId) { + client.elements.fetchMoreElementData(configId); + } + }, [configId, client.elements]); + + React.useEffect(() => { + setState(INITIAL_INCREMENTAL_STATE); + if (configId) { + return client.elements.subscribeToIncrementalElementData( + configId, + chunk => setState(prev => applyElementDataChunk(prev, chunk)), + ); + } + }, [client, configId]); + + return [state.data, loadMore, state.info]; +} + /** * Provides the latest value for entire config or certain key within the config * @param {string} key Key within Plugin Config, optional diff --git a/packages/plugin-sdk/src/types.ts b/packages/plugin-sdk/src/types.ts index f6fecc6..f23f263 100644 --- a/packages/plugin-sdk/src/types.ts +++ b/packages/plugin-sdk/src/types.ts @@ -70,6 +70,26 @@ export interface WorkbookElementData { [colId: string]: any[]; } +/** + * A chunk of rows delivered through an incremental element data subscription. + * Hosts must deliver chunks in non-decreasing offset order (offset 0 restarts + * and replaces all accumulated state), include the subscription's full column + * set in every chunk with all column arrays the same length, and may send an + * empty data object at offset > 0 to update isComplete/totalRows without + * appending rows. + * @typedef {object} WorkbookElementDataChunk + * @property {WorkbookElementData} data Rows contained in this chunk only + * @property {number} offset Absolute row offset of the first row in this chunk; a non-negative integer + * @property {boolean} isComplete True when no more rows are available to fetch + * @property {(number | undefined)} totalRows Total rows in the source element, if known + */ +export interface WorkbookElementDataChunk { + data: WorkbookElementData; + offset: number; + isComplete: boolean; + totalRows?: number; +} + /** * Column data * @typedef {object} WorkbookElementColumn @@ -383,6 +403,29 @@ export interface PluginInstance { callback: (data: WorkbookElementData) => void, ): Unsubscriber; + /** + * Subscriber for the data within a given sheet, delivered incrementally. + * Advertises incremental (append-semantics) delivery to the host: hosts + * that support it deliver each page as a chunk of new rows at an absolute + * row offset, while hosts that do not silently keep sending cumulative + * payloads, which are delivered as replace-everything chunks at offset 0 + * with isComplete false. Callers can treat both hosts identically, but + * must not assume isComplete ever becomes true: hosts without incremental + * support never signal completion. This method defines the plugin half of + * the protocol and behaves like subscribeToElementData against hosts + * without incremental support. Use one subscription style per element: + * the delivery mode belongs to the (plugin, element) subscription, so + * mixing this with subscribeToElementData on the same configId is + * unsupported. + * @param {string} configId ID from config of type: 'element' + * @callback callback Function to call with each chunk of data + * @returns {Unsubscriber} A callable unsubscriber to changes in the data + */ + subscribeToIncrementalElementData( + configId: string, + callback: (chunk: WorkbookElementDataChunk) => void, + ): Unsubscriber; + /** * Ask sigma to load more data * @param {string} configId ID from config of type: 'element'