From 849095d79c004e61c47eb6b8e59412b116cef910 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 3 Sep 2026 10:52:35 -0400 Subject: [PATCH 1/4] =?UTF-8?q?Chunked=20JS=20surface=20=E2=80=94=20types,?= =?UTF-8?q?=20codegen=20spec,=20chunkPlan=20(#32)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the TypeScript surface for chunked uploads. It contains no engine code. The new API is `startUpload` with `type: 'chunked'`. The options are: - `id` — the upload id. The consumer supplies it. It is required. - `path` — the file to send. - `parts` — a list of parts. Each part has a URL, headers, and a byte range. - `accept` — rules that say which HTTP responses are a success. Each rule is a status code and optional body text. These rules replace `acceptStatus` in all upload types, so the library has one shape. - `expiresAt` — the time when the library must stop the upload. It is required. The library validates the parts. The first part must start at byte 0. Each next part must start where the last part ends. The PR adds two functions. `chunkPlan` is a pure function: give it a file size, and it returns the byte ranges for the parts. A property test sweeps every size at a reduced scale. `removeUpload` releases a stopped upload's file and records. Codegen cannot model the raw/chunked union. Thus the native side gets a separate `startChunkedUpload` entry point. The public `startUpload` stays one function and routes on `options.type`. The native functions are stubs on both platforms. They return `E_NOT_IMPLEMENTED`. Thus this PR compiles and passes CI alone. The engine PRs above this one replace the stubs. The PR also turns on `noUncheckedIndexedAccess`. Diana type-checks the shipped `.ts` files with that setting, so the library must catch those breaks first. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 --- .../backgroundupload/UploaderModule.kt | 10 ++ ios/RNFileUploader.mm | 16 ++ src/NativeRNFileUploader.ts | 9 ++ src/__tests__/chunkPlan.test.ts | 99 +++++++++++++ src/__tests__/index.test.ts | 139 +++++++++++++++++- src/chunkPlan.ts | 57 +++++++ src/index.ts | 125 +++++++++++++--- src/types.ts | 59 +++++++- tsconfig.json | 4 + 9 files changed, 493 insertions(+), 25 deletions(-) create mode 100644 src/__tests__/chunkPlan.test.ts create mode 100644 src/chunkPlan.ts diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index 8771af1e..209c89d6 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -265,4 +265,14 @@ class UploaderModule(context: ReactApplicationContext) : promise.reject(exc) } } + + // The chunked transport arrives in the platform changes that follow this + // spec change. Until then, a rejection satisfies the codegen contract. + override fun startChunkedUpload(options: ReadableMap, promise: Promise) { + promise.reject("E_NOT_IMPLEMENTED", "Chunked uploads are not implemented in this build") + } + + override fun removeUpload(id: String, promise: Promise) { + promise.reject("E_NOT_IMPLEMENTED", "removeUpload is not implemented in this build") + } } diff --git a/ios/RNFileUploader.mm b/ios/RNFileUploader.mm index 8c5c7854..ab8c759f 100644 --- a/ios/RNFileUploader.mm +++ b/ios/RNFileUploader.mm @@ -86,6 +86,22 @@ - (void)cancelUpload:(NSString *)id [RNBackgroundUpload.shared cancelUpload:id resolve:resolve reject:reject]; } +// Chunked transport lands in the iOS engine change; until then the spec +// contract is met by rejecting. +- (void)startChunkedUpload:(NSDictionary *)options + resolve:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject +{ + reject(@"E_NOT_IMPLEMENTED", @"Chunked uploads are not implemented in this build", nil); +} + +- (void)removeUpload:(NSString *)id + resolve:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject +{ + reject(@"E_NOT_IMPLEMENTED", @"removeUpload is not implemented in this build", nil); +} + - (void)getUnacknowledgedEvents:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { diff --git a/src/NativeRNFileUploader.ts b/src/NativeRNFileUploader.ts index 99b0df99..ff35ac28 100644 --- a/src/NativeRNFileUploader.ts +++ b/src/NativeRNFileUploader.ts @@ -13,7 +13,16 @@ export interface Spec extends TurboModule { // iOS has no library notification. configure(options: CodegenTypes.UnsafeObject): void; startUpload(options: CodegenTypes.UnsafeObject): Promise; + // Chunked uploads get their own entry point for two reasons. Codegen cannot + // model the raw/chunked discriminated union. And the native implementations + // share no parsing: startUpload dispatches one request, while + // startChunkedUpload creates or reconciles a durable part manifest. index.ts + // keeps the single public startUpload and routes on options.type. + startChunkedUpload(options: CodegenTypes.UnsafeObject): Promise; cancelUpload(id: string): Promise; + // Releases the manifest and the bytes of a non-completed upload. A completed + // upload releases itself when you acknowledge its terminal event. + removeUpload(id: string): Promise; getUnacknowledgedEvents(): Promise; ackEvents(ids: string[]): Promise; getAllUploads(): Promise; diff --git a/src/__tests__/chunkPlan.test.ts b/src/__tests__/chunkPlan.test.ts new file mode 100644 index 00000000..3d7d619b --- /dev/null +++ b/src/__tests__/chunkPlan.test.ts @@ -0,0 +1,99 @@ +import { chunkPlan } from '../chunkPlan'; + +const MB = 2 ** 20; +const MIN = 8 * MB; +const MAX = 20 * MB; + +const assertContiguousCovering = ( + ranges: Array<{ start: number; end: number }>, + size: number, +) => { + expect(ranges.length).toBeGreaterThan(0); + expect(ranges[0].start).toBe(0); + expect(ranges[ranges.length - 1].end).toBe(size); + for (let i = 1; i < ranges.length; i++) { + expect(ranges[i].start).toBe(ranges[i - 1].end); + } +}; + +describe('chunkPlan', () => { + it('is deterministic', () => { + expect(chunkPlan(137 * MB + 3)).toEqual(chunkPlan(137 * MB + 3)); + }); + + it('walks greedy max-size chunks, contiguous and covering', () => { + const size = 100 * MB; + const ranges = chunkPlan(size); + assertContiguousCovering(ranges, size); + expect(ranges).toHaveLength(5); + ranges.forEach((r) => expect(r.end - r.start).toBe(MAX)); + }); + + it('an exact multiple of max yields equal chunks with no absorption', () => { + const ranges = chunkPlan(60 * MB); + expect(ranges).toEqual([ + { start: 0, end: 20 * MB }, + { start: 20 * MB, end: 40 * MB }, + { start: 40 * MB, end: 60 * MB }, + ]); + }); + + it('absorbs a sub-min tail into the previous chunk', () => { + const size = 40 * MB + (MIN - 1); + const ranges = chunkPlan(size); + assertContiguousCovering(ranges, size); + expect(ranges).toHaveLength(2); + // The documented ceiling: the last chunk can reach max + min - 1. + expect(ranges[1].end - ranges[1].start).toBe(MAX + MIN - 1); + }); + + it('keeps a tail of exactly min as its own chunk', () => { + const size = 40 * MB + MIN; + const ranges = chunkPlan(size); + assertContiguousCovering(ranges, size); + expect(ranges).toHaveLength(3); + expect(ranges[2]).toEqual({ start: 40 * MB, end: size }); + }); + + it('a file smaller than min is a single chunk', () => { + expect(chunkPlan(5 * MB)).toEqual([{ start: 0, end: 5 * MB }]); + expect(chunkPlan(1)).toEqual([{ start: 0, end: 1 }]); + }); + + it('throws on size 0 and other non-positive-integer sizes', () => { + expect(() => chunkPlan(0)).toThrow(/positive integer/); + expect(() => chunkPlan(-1)).toThrow(/positive integer/); + expect(() => chunkPlan(1.5)).toThrow(/positive integer/); + expect(() => chunkPlan(NaN)).toThrow(/positive integer/); + }); + + it('throws on nonsensical min/max', () => { + expect(() => chunkPlan(100, { min: 0, max: 20 })).toThrow(/min/); + expect(() => chunkPlan(100, { min: 8, max: 4 })).toThrow(/max/); + }); + + // A scaled-down min/max sweep over every size from 1 byte to 10×max. It + // shows that the shape properties hold at every boundary, not only at the + // sampled sizes above. + it('every plan is contiguous, covering, and never has a tiny part', () => { + const min = 8; + const max = 20; + for (let size = 1; size <= 200; size++) { + const ranges = chunkPlan(size, { min, max }); + assertContiguousCovering(ranges, size); + ranges.forEach((r, i) => { + const len = r.end - r.start; + expect(len).toBeGreaterThan(0); + expect(len).toBeLessThanOrEqual(max + min - 1); + // In a multi-part plan, every part meets the server minimum. Only a + // whole file smaller than min can be under it, as its only part. + if (ranges.length > 1) { + expect(len).toBeGreaterThanOrEqual(min); + } + if (i < ranges.length - 1) { + expect(len).toBe(max); + } + }); + } + }); +}); diff --git a/src/__tests__/index.test.ts b/src/__tests__/index.test.ts index 53087c3f..220fe533 100644 --- a/src/__tests__/index.test.ts +++ b/src/__tests__/index.test.ts @@ -8,7 +8,9 @@ jest.mock('react-native', () => { const nativeModule = { configure: jest.fn(), startUpload: jest.fn(async () => 'id-1'), + startChunkedUpload: jest.fn(async () => 'id-2'), cancelUpload: jest.fn(async () => true), + removeUpload: jest.fn(async () => undefined), getUnacknowledgedEvents: jest.fn(async () => [ { eventId: 'e1', @@ -79,13 +81,13 @@ describe('startUpload', () => { path: '/tmp/f.bin', method: 'POST', type: 'raw', - acceptStatus: [409], + accept: [{ status: 409, bodyIncludes: 'already completed' }], }); expect(native.startUpload).toHaveBeenCalledWith( expect.objectContaining({ url: 'https://example.com/up', path: 'file:///tmp/f.bin', - acceptStatus: [409], + accept: [{ status: 409, bodyIncludes: 'already completed' }], }), ); }); @@ -106,6 +108,139 @@ describe('startUpload', () => { }); }); +describe('startUpload (chunked)', () => { + const chunked = { + type: 'chunked' as const, + id: 'u1', + path: '/tmp/f.bin', + parts: [ + { + url: 'https://example.com/up?partNum=1', + headers: { 'Content-Range': 'bytes 0-9/20' }, + range: { start: 0, end: 10 }, + }, + { + url: 'https://example.com/up?partNum=2', + headers: { 'Content-Range': 'bytes 10-19/20' }, + range: { start: 10, end: 20 }, + }, + ], + expiresAt: 1735689600000, + }; + + it('routes to startChunkedUpload, not startUpload', async () => { + native.startUpload.mockClear(); + await Upload.startUpload(chunked); + expect(native.startUpload).not.toHaveBeenCalled(); + expect(native.startChunkedUpload).toHaveBeenCalledWith( + expect.objectContaining({ + id: 'u1', + path: 'file:///tmp/f.bin', + parts: chunked.parts, + expiresAt: chunked.expiresAt, + }), + ); + }); + + it('rejects an empty id', () => { + expect(() => Upload.startUpload({ ...chunked, id: '' })).toThrow( + /non-empty id/, + ); + }); + + it('rejects empty parts', () => { + expect(() => Upload.startUpload({ ...chunked, parts: [] })).toThrow( + /non-empty/, + ); + }); + + it.each([ + { start: -1, end: 10 }, + { start: 10, end: 10 }, + { start: 11, end: 10 }, + { start: 0.5, end: 10 }, + { start: 0, end: NaN }, + ])('rejects range %p', (range) => { + const parts = [{ ...chunked.parts[0], range }]; + expect(() => Upload.startUpload({ ...chunked, parts })).toThrow( + /0 <= start < end/, + ); + }); + + it.each([NaN, Infinity, 0, -5])('rejects expiresAt %p', (expiresAt) => { + expect(() => Upload.startUpload({ ...chunked, expiresAt })).toThrow( + /expiresAt/, + ); + }); + + it('rejects a nonzero first start', () => { + const parts = [ + { ...chunked.parts[0], range: { start: 5, end: 10 } }, + { ...chunked.parts[1], range: { start: 10, end: 20 } }, + ]; + expect(() => Upload.startUpload({ ...chunked, parts })).toThrow( + /parts\[0\]\.range\.start must be 0/, + ); + }); + + it('rejects a gap between parts', () => { + const parts = [ + { ...chunked.parts[0], range: { start: 0, end: 8 } }, + { ...chunked.parts[1], range: { start: 10, end: 20 } }, + ]; + expect(() => Upload.startUpload({ ...chunked, parts })).toThrow( + /no gaps or overlaps/, + ); + }); + + it('rejects overlapping parts', () => { + const parts = [ + { ...chunked.parts[0], range: { start: 0, end: 12 } }, + { ...chunked.parts[1], range: { start: 10, end: 20 } }, + ]; + expect(() => Upload.startUpload({ ...chunked, parts })).toThrow( + /no gaps or overlaps/, + ); + }); + + it('rejects out-of-order parts', () => { + const parts = [chunked.parts[1], chunked.parts[0]]; + expect(() => Upload.startUpload({ ...chunked, parts })).toThrow( + /parts\[0\]\.range\.start must be 0/, + ); + }); + + it('rejects an empty part url', () => { + const parts = [{ ...chunked.parts[0], url: '' }, chunked.parts[1]]; + expect(() => Upload.startUpload({ ...chunked, parts })).toThrow( + /parts\[0\]\.url must be a non-empty string/, + ); + }); + + it('rejects non-object part headers', () => { + const parts = [ + { ...chunked.parts[0], headers: 'nope' as never }, + chunked.parts[1], + ]; + expect(() => Upload.startUpload({ ...chunked, parts })).toThrow( + /parts\[0\]\.headers must be a plain object/, + ); + }); + + it('throws before reaching native', () => { + native.startChunkedUpload.mockClear(); + expect(() => Upload.startUpload({ ...chunked, parts: [] })).toThrow(); + expect(native.startChunkedUpload).not.toHaveBeenCalled(); + }); +}); + +describe('removeUpload', () => { + it('forwards the id to native', async () => { + await Upload.removeUpload('u1'); + expect(native.removeUpload).toHaveBeenCalledWith('u1'); + }); +}); + describe('addListener', () => { it('subscribes to the matching codegen emitter', () => { Upload.addListener('progress', jest.fn()); diff --git a/src/chunkPlan.ts b/src/chunkPlan.ts new file mode 100644 index 00000000..bcdee924 --- /dev/null +++ b/src/chunkPlan.ts @@ -0,0 +1,57 @@ +export type ChunkPlanOptions = { + /** The smallest chunk that the server accepts in a multi-part upload. */ + min?: number; + /** The target chunk size for the greedy walk. */ + max?: number; +}; + +const DEFAULT_MIN = 8 * 2 ** 20; +const DEFAULT_MAX = 20 * 2 ** 20; + +/** + * Splits `sizeBytes` into contiguous, end-exclusive ranges that cover + * [0, size). The walk is greedy and deterministic: it emits `max`-sized + * chunks. When the final remainder is smaller than `min`, the walk absorbs it + * into the previous chunk. Thus the last chunk can hold up to `max + min - 1` + * bytes. A file smaller than `min` is a single chunk, because the server's + * minimum applies to the parts of a multi-part upload, not to the whole file. + * + * The function is pure and deterministic on purpose. The consumer calls it + * one time, and derives the create request's part count and the `parts` array + * from the same result. Thus the two can never disagree. + */ +export const chunkPlan = ( + sizeBytes: number, + opts: ChunkPlanOptions = {}, +): Array<{ start: number; end: number }> => { + const min = opts.min ?? DEFAULT_MIN; + const max = opts.max ?? DEFAULT_MAX; + if (!Number.isInteger(sizeBytes) || sizeBytes <= 0) { + throw new Error( + `chunkPlan: sizeBytes must be a positive integer, got ${sizeBytes}`, + ); + } + if (!Number.isInteger(min) || min < 1) { + throw new Error(`chunkPlan: min must be a positive integer, got ${min}`); + } + if (!Number.isInteger(max) || max < min) { + throw new Error(`chunkPlan: max must be an integer >= min, got ${max}`); + } + + const ranges: Array<{ start: number; end: number }> = []; + let start = 0; + while (sizeBytes - start >= max) { + ranges.push({ start, end: start + max }); + start += max; + } + const remainder = sizeBytes - start; + if (remainder > 0) { + const last = ranges[ranges.length - 1]; + if (remainder < min && last) { + last.end = sizeBytes; + } else { + ranges.push({ start, end: sizeBytes }); + } + } + return ranges; +}; diff --git a/src/index.ts b/src/index.ts index bdea0e96..28bc49e1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -6,14 +6,17 @@ import type { EventSubscription } from 'react-native'; import NativeRNFileUploader from './NativeRNFileUploader'; import { AddListener, + ChunkedUploadOptions, ConfigureOptions, JournaledEvent, + StartUploadOptions, UploadId, - UploadOptions, UploadSnapshot, } from './types'; +import { chunkPlan } from './chunkPlan'; export * from './types'; +export * from './chunkPlan'; const fileURIPrefix = 'file://'; @@ -29,29 +32,115 @@ const configure = ({ android }: ConfigureOptions): void => { NativeRNFileUploader.configure({ ...android }); }; -/** - * Starts uploading a file to an HTTP endpoint. See UploadOptions for the full - * option set (url, path, method, headers, wifiOnly, acceptStatus, android). - * Returns a promise resolving to the upload's string id. Rejects only on a bad - * option (e.g. missing/invalid url or path); transport failures and HTTP error - * responses surface later as 'error' events, not a rejection here. - */ -const startUpload = ({ - path, - android, - ...options -}: UploadOptions): Promise => { +const normalizePath = (path: string): string => { if (!path.startsWith(fileURIPrefix)) { path = fileURIPrefix + path; } + // Android native takes a plain filesystem path. iOS takes a file:// URL. + return Platform.OS === 'android' ? path.replace(fileURIPrefix, '') : path; +}; - if (Platform.OS === 'android') { - path = path.replace(fileURIPrefix, ''); +// Reject malformed chunked input before it crosses the bridge. Then native +// never creates a manifest for an upload that cannot complete. +const validateChunkedOptions = (options: ChunkedUploadOptions): void => { + if (!options.id) { + throw new Error('startUpload: a chunked upload requires a non-empty id'); } + if (!Array.isArray(options.parts) || options.parts.length === 0) { + throw new Error('startUpload: parts must be a non-empty array'); + } + // The parts must tile the file from byte 0. They must be sorted in + // ascending order, with no gaps and no overlaps. Each plan from chunkPlan + // obeys this by construction. + let expectedStart = 0; + options.parts.forEach(({ url, headers, range }, i) => { + if (typeof url !== 'string' || url.length === 0) { + throw new Error(`startUpload: parts[${i}].url must be a non-empty string`); + } + if ( + headers !== undefined && + (typeof headers !== 'object' || + headers === null || + Array.isArray(headers)) + ) { + throw new Error( + `startUpload: parts[${i}].headers must be a plain object when present`, + ); + } + if ( + !range || + !Number.isInteger(range.start) || + !Number.isInteger(range.end) || + range.start < 0 || + range.start >= range.end + ) { + throw new Error( + `startUpload: parts[${i}].range must satisfy 0 <= start < end, got ${JSON.stringify( + range, + )}`, + ); + } + if (range.start !== expectedStart) { + throw new Error( + i === 0 + ? `startUpload: parts[0].range.start must be 0, got ${range.start}` + : `startUpload: parts must be sorted ascending and tile the file with no gaps or overlaps — parts[${i}].range.start is ${range.start} but parts[${i - 1}].range.end is ${expectedStart}`, + ); + } + expectedStart = range.end; + }); + if (!Number.isFinite(options.expiresAt) || options.expiresAt <= 0) { + throw new Error( + `startUpload: expiresAt must be a finite epoch-ms timestamp, got ${options.expiresAt}`, + ); + } +}; - return NativeRNFileUploader.startUpload({ ...options, ...android, path }); +/** + * Starts an upload to an HTTP endpoint. The behavior depends on options.type. + * 'raw' sends the whole file as one request body. 'chunked' sends the parts + * that the consumer authored (see ChunkedUploadOptions). Returns a promise + * that resolves to the upload's string id. Malformed chunked input throws + * synchronously. Other bad options (for example, a missing or invalid url or + * path) reject. Transport failures and HTTP error responses arrive later as + * 'error' events. + * + * A new call with the same id and identical parts is never an error, at any + * time. The library reconciles: it skips the parts that the server accepted, + * and the other parts continue with the new call's headers. A call with a + * different parts array is a recreate. The library accepts a recreate when + * the upload is stopped, and replaces the bytes. It rejects a recreate while + * the upload runs. + */ +const startUpload = (options: StartUploadOptions): Promise => { + if (options.type === 'chunked') { + validateChunkedOptions(options); + const { path, android, ...rest } = options; + return NativeRNFileUploader.startChunkedUpload({ + ...rest, + ...android, + path: normalizePath(path), + }); + } + + const { path, android, ...rest } = options; + return NativeRNFileUploader.startUpload({ + ...rest, + ...android, + path: normalizePath(path), + }); }; +/** + * Releases an upload's native manifest and bytes. Each terminal outcome other + * than an acknowledged 'completed' (expired, error, cancelled) keeps both. + * This lets the consumer resume or recreate the upload. Call this function + * when you want neither. On a raw upload id, it cancels the in-flight request + * and deliberately emits no terminal event. + */ +const removeUpload = (uploadId: string): Promise => + NativeRNFileUploader.removeUpload(uploadId); + /** * Cancels active upload by string ID of the upload. * @@ -101,7 +190,7 @@ const addListener = (( * startup, process each, then acknowledge — unacknowledged events are re-delivered * here on every call until you ack them. * - * Note: `completed` fires only for 2xx (or a request's `acceptStatus`); other HTTP + * Note: `completed` fires only for 2xx (or a request's `accept` rules); other HTTP * responses arrive as `error` with `errorKind: 'http'` and the response attached. */ const getUnacknowledgedEvents = async (): Promise => @@ -134,9 +223,11 @@ export default { configure, startUpload, cancelUpload, + removeUpload, addListener, getUnacknowledgedEvents, ackEvents, getAllUploads, + chunkPlan, android, }; diff --git a/src/types.ts b/src/types.ts index 0429edac..620b317e 100644 --- a/src/types.ts +++ b/src/types.ts @@ -8,7 +8,12 @@ export interface ProgressData extends EventData { progress: number; } -export type ErrorKind = 'http' | 'network' | 'file' | 'unknown'; +/** + * `expired` means that the upload's `expiresAt` time passed before the server + * accepted every part. The library keeps the manifest and the bytes. Thus a + * new `startUpload` call with a later deadline resumes the upload. + */ +export type ErrorKind = 'http' | 'network' | 'file' | 'expired' | 'unknown'; export type CancelReason = 'user' | 'system'; @@ -38,7 +43,7 @@ export interface TerminalEventData extends EventData { responseHeaders?: Record; } -/** A 2xx response, or one whose status was listed in the request's `acceptStatus`. */ +/** A 2xx response, or one that matched an `accept` rule on the request. */ export interface CompletedData extends TerminalEventData { type: 'completed'; } @@ -85,15 +90,57 @@ export type UploadOptions = { }; // Whether the upload should wait for wifi before starting wifiOnly?: boolean; - // Non-2xx statuses to treat as a successful completion (e.g. [409] when - // duplicate-create conflicts are expected). Anything else non-2xx emits an - // 'error' event with errorKind 'http'. - acceptStatus?: number[]; + accept?: AcceptRule[]; // Android options that change behavior. Notification text is not a // per-upload option. Set it one time with configure(). android?: Partial; } & RawUploadOptions; +/** + * A non-2xx response to treat as success. `bodyIncludes` narrows the rule by + * a response-body substring. This is necessary when one status has several + * meanings, and only the message shows the difference (our backend's 409). A + * non-2xx response that matches no rule emits an 'error' event with errorKind + * 'http'. + */ +export type AcceptRule = { + status: number; + bodyIncludes?: string; +}; + +export type ChunkedUploadOptions = { + type: 'chunked'; + /** Required. The consumer's durable id. */ + id: string; + /** + * The single source file. The library takes ownership: at startUpload it + * renames the file into the library's own directory (an O(1) move). It + * deletes the file only after you acknowledge a 'completed' terminal event. + * If you must keep the file, copy it first. A keep-the-file mode is + * deliberately not part of the library. + */ + path: string; + /** + * The consumer authors this one time. The library sends the file bytes + * [range.start, range.end) as the body of a PUT to `url`, with `headers` + * unchanged. The library never derives or edits a protocol field. + */ + parts: Array<{ + url: string; + /** These headers include Content-Range, Content-Type, and auth. */ + headers: Record; + /** Byte offsets. The end is exclusive. */ + range: { start: number; end: number }; + }>; + accept?: AcceptRule[]; + /** Epoch ms. Required. After this time: terminal error, errorKind 'expired'. */ + expiresAt: number; + wifiOnly?: boolean; + android?: Partial; +}; + +export type StartUploadOptions = UploadOptions | ChunkedUploadOptions; + export type AndroidOnlyUploadOptions = { /** * Uploads this file without a progress notification. Default false. diff --git a/tsconfig.json b/tsconfig.json index 1a8567cd..f2d65945 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -12,6 +12,10 @@ "strictNullChecks": true, "skipLibCheck": true, "noImplicitAny": true, + // Diana type-checks this package's shipped .ts sources with its own + // stricter configuration. Match that configuration here. Then breaks + // appear here, not in Diana. + "noUncheckedIndexedAccess": true, "noEmit": true, // Don't auto-include every @types/* in node_modules: src needs no ambient // types (it imports react-native's explicitly, and tests are excluded), and From 2860e9599b1ddd72e0f46ac8c25161f67369780e Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 3 Sep 2026 10:52:36 -0400 Subject: [PATCH 2/4] Android chunked-upload engine (#33) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the Android engine for chunked uploads. At start, the library moves the file into its own directory. The library then owns the file. The library writes a durable manifest to disk. The manifest holds the parts, the status of each part, and the options. The engine sends the parts with WorkManager. A RandomAccessFile-backed body streams each part directly from the file. There are no chunk copy files, and disk usage does not double. A maximum of 3 parts transmit at one time (the window). The engine applies the accept rules and the expiry time. The engine writes each event to a journal before it emits the event. The library deletes the file only when the consumer acknowledges the `completed` event. `startUpload` with the same id resumes the upload. The engine skips the parts that the server accepted. The same id with different parts recreates the upload. The engine permits a recreate only when the upload is not running, and the new parts must tile the file exactly. Review fixes in this slice: - A job that v8 enqueued replays safely. The engine normalizes the legacy `acceptStatus` field. A test uses the literal v8 JSON shape. - The reconcile step is atomic under the store lock. - The work policy is `APPEND_OR_REPLACE`, and a worker returns success after it journals a terminal error. Thus a failing chain cannot silently drop a resume. - An ack race resolves cleanly. A missing manifest is a no-op. - A range-tiling guard protects the recreate rule. 84 JVM tests. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 --- android/consumer-rules.pro | 14 +- .../backgroundupload/ChunkedEngine.kt | 124 ++++++ .../backgroundupload/ChunkedManifest.kt | 298 +++++++++++++ .../backgroundupload/ChunkedUploadWorker.kt | 410 ++++++++++++++++++ .../backgroundupload/ChunkedWorkerGate.kt | 40 ++ .../backgroundupload/EventJournal.kt | 6 +- .../backgroundupload/EventReporter.kt | 9 + .../ai/openspace/backgroundupload/Upload.kt | 72 ++- .../backgroundupload/UploadNotification.kt | 135 ++++++ .../backgroundupload/UploadOutcome.kt | 25 +- .../backgroundupload/UploadTransport.kt | 28 ++ .../openspace/backgroundupload/UploadUtils.kt | 98 ++++- .../backgroundupload/UploadWorker.kt | 184 ++------ .../backgroundupload/UploaderModule.kt | 322 ++++++++++++-- .../backgroundupload/AckReleaseTest.kt | 69 +++ .../backgroundupload/ChunkedEngineTest.kt | 181 ++++++++ .../backgroundupload/ChunkedManifestTest.kt | 384 ++++++++++++++++ .../backgroundupload/ChunkedWorkerGateTest.kt | 57 +++ .../backgroundupload/UploadOutcomeTest.kt | 48 +- .../backgroundupload/UploadStatesTest.kt | 84 ++++ .../openspace/backgroundupload/UploadTest.kt | 71 ++- src/types.ts | 11 +- 22 files changed, 2419 insertions(+), 251 deletions(-) create mode 100644 android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt create mode 100644 android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt create mode 100644 android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt diff --git a/android/consumer-rules.pro b/android/consumer-rules.pro index c831f56b..754b74ca 100644 --- a/android/consumer-rules.pro +++ b/android/consumer-rules.pro @@ -9,13 +9,14 @@ # collections. Thus, if R8 renames a field or removes Signature, it corrupts the # persisted state silently: # -# * Upload.acceptStatus is a List. Without Signature, Gson decodes it as -# List. Then acceptStatus.contains(code) never matches, and a -# configured accept status (for example 409) is reported as an http error, -# not as a completed upload. +# * Upload.accept is a List. Without Signature, Gson decodes the +# elements as bare maps. Then no rule ever matches, and a configured accept +# status (for example 409) is reported as an http error, not as a completed +# upload. ChunkedManifest.parts has the same shape and the same failure. # * A journal Entry from an older build fails to parse if field names changed. # The library then drops the Entry as malformed. This loses the terminal -# outcomes that the journal exists to keep. +# outcomes that the journal exists to keep. A ChunkedManifest is the resume +# record for a chunked upload, and it fails in the same way. # # Debug builds are not minified and round-trip correctly. Thus neither failure # is reproducible without R8. Keep these rules. @@ -26,3 +27,6 @@ -keep class ai.openspace.backgroundupload.Upload$* { *; } -keep class ai.openspace.backgroundupload.NotificationConfig { *; } -keep class ai.openspace.backgroundupload.EventJournal$Entry { *; } +-keep class ai.openspace.backgroundupload.ChunkedManifest { *; } +-keep class ai.openspace.backgroundupload.ChunkedManifest$* { *; } +-keep class ai.openspace.backgroundupload.UploadOutcome$AcceptRule { *; } diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt new file mode 100644 index 00000000..e398c750 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedEngine.kt @@ -0,0 +1,124 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.coroutineScope +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit + +/** + * The pure scheduling half of chunked execution: the window, the retry + * policy, and the backoff. It is kept free of Android and OkHttp types. Thus + * the highest-consequence invariants (at most WINDOW parts in flight, and + * never two requests for one part index) are unit-testable on a plain JVM. + * [ChunkedUploadWorker] supplies the part executor. + */ +object ChunkedEngine { + + // The number of parts of one upload in flight at one time. This is a library + // constant, not an option. If soak data shows that a different value is + // better, this constant changes, not the API. + const val WINDOW = 3 + + // The library retries a non-accepted, non-transient HTTP response this many + // times per part. Then the response becomes a terminal error and stalls the + // upload. The budget is small on purpose. A response that the server repeats + // (401, 400) does not change without a new startUpload. Only transient + // failures retry without a limit. + const val PART_HTTP_RETRIES = 3 + + // The poll interval while the network is unusable (offline, or waiting for + // wifi). The interval is constant, not exponential. We wait for conditions + // here; we do not back off a server. And expiresAt bounds the total wait. + const val CONNECTIVITY_POLL_MS = 10_000L + + private const val BACKOFF_BASE_MS = 1_000L + private const val BACKOFF_CAP_MS = 60_000L + + // A 5xx means that the server failed, not that the request is wrong. Thus it + // retries like a transport failure: without a limit, until expiresAt. + fun isTransientHttp(code: Int) = code in 500..599 + + /** What a starting worker must do for the manifest that it finds (or does not find). */ + enum class StartAction { + /** + * No manifest exists. The upload was completed and acknowledged, or it was + * explicitly removed, while this run sat in the queue. Both are legitimate + * ends, already reported (or deliberately not reported). Exit with success + * and in silence. A journaled terminal here would be a spurious 'file' + * error for an upload that nobody owns any more. + */ + NO_MANIFEST, + + /** + * Every part is already accepted: this is a trailing resume run. Re-report + * the journaled completion (never mint a second terminal event) and stop. + * Start no foreground service and no transfers. + */ + ALREADY_COMPLETE, + + /** Pending parts remain. Run the engine. */ + RUN, + } + + fun startAction(manifest: ChunkedManifest?): StartAction = when { + manifest == null -> StartAction.NO_MANIFEST + manifest.allAccepted -> StartAction.ALREADY_COMPLETE + else -> StartAction.RUN + } + + /** How a run that found (or produced) an all-accepted manifest reports the completion. */ + sealed class CompletionReport { + /** An unacknowledged 'completed' entry exists. Re-emit it. Never mint a second entry. */ + data class ReEmit(val entry: EventJournal.Entry) : CompletionReport() + + /** A fresh completion with no journal entry yet. Journal and emit a new entry. */ + object Mint : CompletionReport() + + /** + * A trailing run with nothing unacknowledged: the completion was journaled + * AND acknowledged. Nobody is owed an event. This occurs when the trailing + * run races ackEvents, which deletes the journal entry just before the + * manifest. An event minted here would be a duplicate 'completed' for an + * upload that the consumer already settled. + */ + object None : CompletionReport() + } + + fun completionReport( + unacked: List, + uploadId: String, + freshCompletion: Boolean, + ): CompletionReport { + val existing = unacked.firstOrNull { it.uploadId == uploadId && it.type == "completed" } + return when { + existing != null -> CompletionReport.ReEmit(existing) + freshCompletion -> CompletionReport.Mint + else -> CompletionReport.None + } + } + + /** Exponential backoff for transient failures: 1s, 2s, 4s, and more, capped at 60s. */ + fun backoffMs(attempt: Int): Long = + (BACKOFF_BASE_MS shl (attempt - 1).coerceIn(0, 6)).coerceAtMost(BACKOFF_CAP_MS) + + /** + * Runs [executePart] exactly one time per index, with at most [window] parts + * at one time. One coroutine per part index is what guarantees that no two + * requests for the same part are in flight (concurrent PUTs of one partNum + * are verified unsafe on the server side). An executor that throws cancels + * the remaining parts, and the error propagates. Terminal classification is + * the caller's job. + */ + suspend fun run( + partIndexes: List, + window: Int = WINDOW, + executePart: suspend (Int) -> Unit, + ) { + val gate = Semaphore(window) + coroutineScope { + for (index in partIndexes) { + launch { gate.withPermit { executePart(index) } } + } + } + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt new file mode 100644 index 00000000..6bbe523b --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedManifest.kt @@ -0,0 +1,298 @@ +package ai.openspace.backgroundupload + +import android.content.Context +import com.facebook.react.bridge.ReadableMap +import com.google.gson.Gson +import java.io.File +import java.util.Base64 + +/** + * The durable record of one chunked upload: the moved source file, the parts + * that the consumer authored, and which of them the server has accepted. + * [ChunkedManifestStore] persists it as JSON at startUpload, BEFORE the work + * is enqueued. Thus a worker rescheduled after process death (or a startUpload + * after a crash, a stop, or a reauth) resumes from it without a call into JS. + * This manifest IS the resume mechanism. + * + * The data shape is kept free of Android and React types (Gson round-trips + * it, and JVM tests construct it directly). The ReadableMap parsing lives in + * the companion, like [Upload]'s. + */ +data class ChunkedManifest( + val id: String, + /** The library-owned copy of the bytes (the consumer's file, renamed in). */ + val sourcePath: String, + val parts: List, + val accept: List, + /** Epoch ms. After this time, the upload stops with errorKind 'expired'. */ + val expiresAt: Long, + val wifiOnly: Boolean, + val noNotification: Boolean, + val createdAt: Long, +) { + /** + * One part, exactly as the consumer authored it. The library sends the file + * bytes [start, end) as the body of a PUT to [url], with [headers] + * unchanged. It never derives or edits a protocol field. + */ + data class Part( + val url: String, + val headers: Map, + val start: Long, + val end: Long, // exclusive + val accepted: Boolean = false, + ) { + val size get() = end - start + } + + val showsNotification get() = !noNotification + val totalBytes get() = parts.sumOf { it.size } + val acceptedBytes get() = parts.filter { it.accepted }.sumOf { it.size } + + /** The server's auto-publish condition. It is the only thing that 'completed' may mean. */ + val allAccepted get() = parts.all { it.accepted } + + fun isExpired(now: Long) = now >= expiresAt + + fun pendingIndexes() = parts.indices.filter { !parts[it].accepted } + + fun withPartAccepted(index: Int) = copy( + parts = parts.mapIndexed { i, part -> if (i == index) part.copy(accepted = true) else part }, + ) + + class ReconcileException(message: String) : IllegalArgumentException(message) + + /** + * A startUpload re-call with an existing id is one of two things: + * + * **Resume** — the incoming parts are the SAME array (identical count, + * ranges, and urls). The headers, the accept rules, expiresAt, and the flags + * come from the new call. This is how fresh auth reaches stalled parts, and + * how a salvage extends the deadline. The accepted part statuses, the moved + * source, and createdAt survive from this manifest. A resume is permitted at + * any time, running or not. A running worker re-reads the stored copy before + * every attempt. + * + * **Recreate** — a DIFFERENT parts array. The consumer re-authored the + * upload under a fresh server uploadId after the old one died (it expired + * past the server's 31-day window, or it is otherwise unrecoverable). The + * owned bytes are kept. The parts are replaced as a whole, and every part + * status resets to unsent. The headers, the accept rules, and expiresAt come + * from the new call. The new ranges must tile exactly [0, blobSize). A + * partial or overlapping cover would silently upload wrong bytes. A recreate + * is accepted only while the upload is NOT running (stalled on a terminal + * error, expired, or cancelled). A different parts array while a worker + * executes is a consumer bug, not a recreate, because the in-flight requests + * belong to the old parts. + */ + fun reconcile(incoming: ChunkedManifest, running: Boolean, blobSize: Long): ChunkedManifest { + if (samePartsAs(incoming)) { + // Accepted flags follow the RANGE, not the array index. samePartsAs is + // order-independent, so the same tile can sit at a different index. + val acceptedStarts = parts.filter { it.accepted }.map { it.start }.toSet() + return incoming.copy( + sourcePath = sourcePath, + createdAt = createdAt, + parts = incoming.parts.map { it.copy(accepted = it.start in acceptedStarts) }, + ) + } + if (running) throw ReconcileException( + "chunked upload '$id' is running; a different parts array is only accepted once it stops", + ) + if (!tilesExactly(incoming.parts, blobSize)) throw ReconcileException( + "chunked upload '$id' recreate parts must tile exactly [0, $blobSize)", + ) + return incoming.copy(sourcePath = sourcePath, createdAt = createdAt) + } + + // Order-independent, like tilesExactly. The same tiles, authored in a + // different order, are the SAME upload (a resume), never a recreate. + private fun samePartsAs(incoming: ChunkedManifest): Boolean { + if (incoming.parts.size != parts.size) return false + val stored = parts.sortedBy { it.start } + val fresh = incoming.parts.sortedBy { it.start } + return stored.indices.all { i -> + fresh[i].url == stored[i].url && + fresh[i].start == stored[i].start && + fresh[i].end == stored[i].end + } + } + + companion object { + /** + * Whether [parts] cover [0, size) exactly: no gap, no overlap, and nothing + * past the end. Order-independent, like everything else about parts. + */ + fun tilesExactly(parts: List, size: Long): Boolean { + if (parts.isEmpty()) return false + val sorted = parts.sortedBy { it.start } + var cursor = 0L + for (part in sorted) { + if (part.start != cursor || part.end <= part.start) return false + cursor = part.end + } + return cursor == size + } + + /** + * Validates a first-call (create) manifest against the just-owned bytes. + * Like a recreate, the parts must tile exactly [0, blobSize). A partial or + * overlapping cover would silently upload wrong bytes. It throws BEFORE + * the manifest is saved. Thus the moved blob stays adoptable by a + * corrected retry (see UploaderModule.takeOwnership's orphan branch). + */ + fun validatedForCreate(incoming: ChunkedManifest, blobSize: Long): ChunkedManifest { + if (!tilesExactly(incoming.parts, blobSize)) throw ReconcileException( + "chunked upload '${incoming.id}' parts must tile exactly [0, $blobSize)", + ) + return incoming + } + + /** @param sourcePath the library-owned destination, not the consumer's path. */ + fun fromReadableMap(map: ReadableMap, sourcePath: String, createdAt: Long): ChunkedManifest { + val partsArr = map.getArray("parts") ?: throw Upload.MissingOptionException("parts") + if (partsArr.size() == 0) throw IllegalArgumentException("parts must be a non-empty array") + if (!map.hasKey("expiresAt")) throw Upload.MissingOptionException("expiresAt") + return ChunkedManifest( + id = map.getString("id") ?: throw Upload.MissingOptionException("id"), + sourcePath = sourcePath, + parts = (0 until partsArr.size()).map { i -> + val part = partsArr.getMap(i) ?: throw Upload.MissingOptionException("parts[$i]") + val range = part.getMap("range") ?: throw Upload.MissingOptionException("parts[$i].range") + Part( + url = part.getString("url") ?: throw Upload.MissingOptionException("parts[$i].url"), + headers = parseHeaderMap(part.getMap("headers")), + start = range.getDouble("start").toLong(), + end = range.getDouble("end").toLong(), + ) + }, + accept = parseAcceptRules(map.getArray("accept")), + expiresAt = map.getDouble("expiresAt").toLong(), + wifiOnly = if (map.hasKey("wifiOnly")) map.getBoolean("wifiOnly") else false, + noNotification = if (map.hasKey("noNotification")) map.getBoolean("noNotification") else false, + createdAt = createdAt, + ) + } + } +} + +/** + * A file-backed store: one directory per upload id, which holds + * `manifest.json` and `blob` (the moved source bytes). It has the same + * durability pattern as [EventJournal]: tmp+rename writes, and corrupt files + * read as absent. It is reachable from a bare Context, because the worker can + * run in a process where React never initialized. + */ +class ChunkedManifestStore(private val dir: File) { + + companion object { + private val gson = Gson() + + @Volatile + private var instance: ChunkedManifestStore? = null + + fun get(context: Context): ChunkedManifestStore = + instance ?: synchronized(this) { + instance ?: ChunkedManifestStore(File(context.filesDir, "rnbgupload-chunked")) + .also { instance = it } + } + } + + init { + dir.mkdirs() + } + + // Upload ids come from the consumer, and they can contain path separators or + // other filesystem-hostile characters. Thus the directory name is an encoding + // of the id, never the id itself. The id is read back from the manifest, not + // decoded from the name. + private fun uploadDir(id: String) = + File(dir, Base64.getUrlEncoder().withoutPadding().encodeToString(id.toByteArray())) + + private fun manifestFile(id: String) = File(uploadDir(id), "manifest.json") + + /** Where startUpload moves the source file for this id. */ + fun blobFile(id: String) = File(uploadDir(id), "blob") + + @Synchronized + fun load(id: String): ChunkedManifest? { + val file = manifestFile(id) + if (!file.exists()) return null + val parsed = runCatching { gson.fromJson(file.readText(), ChunkedManifest::class.java) } + .getOrNull() + return validated(parsed) + } + + /** Throws on a write failure. A manifest that did not persist must fail the startUpload call. */ + @Synchronized + fun save(manifest: ChunkedManifest) { + val dir = uploadDir(manifest.id) + dir.mkdirs() + val tmp = File(dir, "manifest.tmp") + tmp.writeText(gson.toJson(manifest)) + if (!tmp.renameTo(manifestFile(manifest.id))) { + throw java.io.IOException("failed to persist chunked manifest for '${manifest.id}'") + } + } + + /** + * An atomic read-modify-write. Thus a worker that marks a part accepted can + * never clobber a concurrent startUpload's fresh headers (or another part's + * flag). Returns null, without a throw, when the manifest is gone or the + * write failed. A caller that can continue from memory does that. + */ + @Synchronized + fun update(id: String, transform: (ChunkedManifest) -> ChunkedManifest): ChunkedManifest? = + runCatching { + val manifest = load(id) ?: return null + val next = transform(manifest) + save(next) + next + }.getOrNull() + + /** + * An atomic create-or-transform. The store lock spans load, [transform], and + * save. Thus nothing — a running worker's markAccepted included — can write + * between them and be erased. startUpload's load, reconcile, and save must + * go through here, not as three separate calls. [transform] receives null + * when no manifest exists. Unlike [update], a transform that throws (a + * reconcile rejection) or a failed write propagates, because startUpload + * must fail loudly, not continue from memory. + */ + @Synchronized + fun compute(id: String, transform: (ChunkedManifest?) -> ChunkedManifest): ChunkedManifest { + val next = transform(load(id)) + save(next) + return next + } + + /** Whether a manifest is stored for this id (without parsing it). */ + @Synchronized + fun contains(id: String): Boolean = manifestFile(id).exists() + + /** Deletes the manifest AND the moved bytes. Does nothing for an unknown id (a simple upload). */ + @Synchronized + fun remove(id: String) { + uploadDir(id).deleteRecursively() + } + + @Synchronized + fun all(): List = + (dir.listFiles { f -> f.isDirectory } ?: emptyArray()) + .mapNotNull { d -> + val file = File(d, "manifest.json") + if (!file.exists()) return@mapNotNull null + validated(runCatching { gson.fromJson(file.readText(), ChunkedManifest::class.java) }.getOrNull()) + } + + // Gson does not use the constructor. Thus a corrupt or field-renamed file can + // make non-null Kotlin fields null. Reject a file that lacks a field that the + // engine relies on. Normalize an absent accept list; do not reject it. + @Suppress("SENSELESS_COMPARISON") + private fun validated(m: ChunkedManifest?): ChunkedManifest? { + if (m == null || m.id == null || m.sourcePath == null || m.parts == null) return null + if (m.parts.isEmpty()) return null + if (m.parts.any { it == null || it.url == null || it.headers == null }) return null + return if (m.accept == null) m.copy(accept = listOf()) else m + } +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt new file mode 100644 index 00000000..33750073 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedUploadWorker.kt @@ -0,0 +1,410 @@ +package ai.openspace.backgroundupload + +import android.app.NotificationManager +import android.content.Context +import androidx.work.CoroutineWorker +import androidx.work.ForegroundInfo +import androidx.work.ListenableWorker +import androidx.work.WorkerParameters +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.sync.withPermit +import kotlinx.coroutines.withContext +import java.io.File +import java.io.IOException +import java.util.UUID +import java.util.concurrent.ConcurrentHashMap +import java.util.concurrent.atomic.AtomicLong + +/** + * Executes one chunked upload from its durable [ChunkedManifest]. The input + * data carries only the upload id. The manifest is the record: startUpload + * persists it before this work is enqueued. Thus a worker rescheduled after + * process death resumes from disk, with no JS involved. + * + * One logical upload has one event stream: byte-weighted aggregate progress, + * and one terminal event. 'completed' is journaled only when every part is + * accepted. Every other terminal keeps the manifest and the bytes, so a later + * startUpload can resume. The bytes are deleted only when a 'completed' event + * is ACKED (see UploaderModule.ackEvents). + */ +class ChunkedUploadWorker(private val context: Context, params: WorkerParameters) : + CoroutineWorker(context, params) { + + companion object { + /** + * The key for the upload id in the worker's input data. It is a string + * literal for the same reason as [UploadWorker.PARAMS_KEY]: WorkManager's + * database persists it across builds, and it must survive R8 renames and + * refactors. + */ + const val ID_KEY = "chunkedUploadId" + + /** How often a starting worker re-checks [ChunkedWorkerGate] for its id. */ + private const val GATE_POLL_MS = 100L + } + + private lateinit var uploadId: String + private val store by lazy { ChunkedManifestStore.get(context) } + private val config by lazy { NotificationConfig.load(context) } + private val notificationManager = + context.getSystemService(Context.NOTIFICATION_SERVICE) as NotificationManager + + // The latest known manifest. Part executors re-read the stored copy before + // every attempt (see latest()). Thus a reconciling startUpload's fresh + // headers, and an extended expiresAt, reach a worker that already runs. + @Volatile + private var manifest: ChunkedManifest? = null + + @Volatile + private var connectivity = Connectivity.Ok + + // In-flight bytes per part index, for byte-weighted aggregate progress. + private val partSent = ConcurrentHashMap() + private val acceptedBytes = AtomicLong(0) + + private class ExpiredException : Exception("upload expired") + + private class SourceMissingException(path: String) : + IOException("chunked source file missing: $path") + + private class PartRejectedException(val partIndex: Int, val response: UploadResponse) : + Exception("part $partIndex rejected with HTTP ${response.code}") + + private class PartBeyondEofException(val partIndex: Int, message: String) : Exception(message) + + override suspend fun doWork(): Result = withContext(Dispatchers.IO) { + uploadId = inputData.getString(ID_KEY) ?: throw Throwable("No upload id") + + // Acquire the per-id execution gate BEFORE the first manifest read. A + // cancel-then-start can start this worker while the cancelled one still + // winds down, and two PUTs of one partNum are unsafe. Also, the manifest + // read occurs only after the gate is held. That is what makes the module's + // recreate check race-free (see ChunkedWorkerGate and + // ChunkedManifestStore.compute). + try { + while (!ChunkedWorkerGate.tryAcquire(uploadId, this@ChunkedUploadWorker)) { + delay(GATE_POLL_MS) + } + } catch (error: CancellationException) { + // Cancelled while waiting. A user cancel still owes its terminal event. + checkAndHandleCancellation() + throw error + } + try { + runUpload() + } finally { + ChunkedWorkerGate.release(uploadId, this@ChunkedUploadWorker) + } + } + + private suspend fun runUpload(): Result { + val initial = store.load(uploadId) + when (ChunkedEngine.startAction(initial)) { + // The upload was completed-and-acknowledged, or it was removed, while + // this run sat in the queue. Both are legitimate and already settled. + // Exit in silence. A terminal journaled here would be a spurious error + // for an upload that nobody owns. + ChunkedEngine.StartAction.NO_MANIFEST -> return Result.success() + // A trailing resume of a finished-but-unacknowledged upload. Re-report + // the journaled completion. Skip the foreground service and the engine. + ChunkedEngine.StartAction.ALREADY_COMPLETE -> { + manifest = initial + journalCompleted(freshCompletion = false) + return Result.success() + } + ChunkedEngine.StartAction.RUN -> Unit + } + checkNotNull(initial) // RUN implies a manifest + manifest = initial + acceptedBytes.set(initial.acceptedBytes) + UploadProgress.add(uploadId, initial.totalBytes) + UploadProgress.set(uploadId, initial.acceptedBytes) + + // Initialization. A failure here is terminal: journaled, never retried. + // The EXCEPTION is a refused foreground start, which the transfer + // survives. + try { + if (initial.showsNotification) { + ensureNotificationChannel(notificationManager, config) + setForeground(getForegroundInfo()) + } + } catch (error: Throwable) { + if (!isForegroundStartDenied(error)) { + if (!checkAndHandleCancellation()) { + UploadProgress.remove(uploadId) + handleFailure(error) + } + return terminalErrorResult() + } + // The app is in the background, and API 31+ refused the foreground + // start. This is the usual state for a WorkManager relaunch (a reboot, + // or a quota resume). The upload runs correctly without foreground + // priority. A failure here would brick every headless resume. + } + + return try { + ChunkedEngine.run(initial.pendingIndexes()) { index -> executePart(index) } + // Every executor returned. An executor returns only when its part was + // accepted. That is exactly the server's auto-publish condition. + UploadProgress.complete(uploadId) + journalCompleted(freshCompletion = true) + Result.success() + } catch (error: Throwable) { + if (checkAndHandleCancellation()) throw error + UploadProgress.remove(uploadId) + handleFailure(error) + terminalErrorResult() + } + } + + /** + * Uploads one part until it is accepted, or throws. Terminal conditions + * (expiry, a missing source, or a non-accepted response out of retries) + * propagate and cancel the sibling parts. Everything transient retries here, + * bounded only by expiresAt. + */ + private suspend fun executePart(index: Int) { + var rejections = 0 + var transientAttempts = 0 + while (true) { + val current = latest() + val part = current.parts[index] + if (part.accepted) return + if (current.isExpired(System.currentTimeMillis())) throw ExpiredException() + + // A range past the blob's EOF can never transmit. The read would fail + // on every attempt until expiry. Thus it is a terminal 'file' error + // immediately (iOS classifies it the same way). length() is 0 for a + // missing file. That case falls through to the transfer, which + // classifies it as source-missing. The failed-probe-reads-as-network + // default stays intact. + val blobLength = runCatching { File(current.sourcePath).length() }.getOrDefault(0L) + if (blobLength > 0L && part.end > blobLength) throw PartBeyondEofException( + index, + "part $index range [${part.start}, ${part.end}) exceeds source size $blobLength", + ) + + if (!validateAndReportConnectivity(current.wifiOnly)) { + delay(ChunkedEngine.CONNECTIVITY_POLL_MS) + continue + } + + val response = try { + transferSemaphore.withPermit { + okhttpUploadPart(uploadHttpClient, part, File(current.sourcePath)) { sent -> + onPartProgress(index, sent) + } + } + } catch (error: CancellationException) { + throw error + } catch (error: IOException) { + onPartProgress(index, 0L) + // The default is fileExists=true. Thus a failed probe reads as + // network, not file. + val fileExists = runCatching { File(current.sourcePath).exists() }.getOrDefault(true) + if (!fileExists) throw SourceMissingException(current.sourcePath) + transientAttempts++ + delay(ChunkedEngine.backoffMs(transientAttempts)) + continue + } + + if (UploadOutcome.isAccepted(response.code, response.body, current.accept)) { + markAccepted(index) + return + } + onPartProgress(index, 0L) + if (ChunkedEngine.isTransientHttp(response.code)) { + transientAttempts++ + delay(ChunkedEngine.backoffMs(transientAttempts)) + continue + } + rejections++ + if (rejections > ChunkedEngine.PART_HTTP_RETRIES) throw PartRejectedException(index, response) + delay(ChunkedEngine.backoffMs(rejections)) + } + } + + // The stored copy is the truth: a reconcile can have replaced the headers + // or expiresAt. Fall back to the in-memory copy only when the read fails. + private fun latest(): ChunkedManifest = + store.load(uploadId)?.also { manifest = it } ?: manifest!! + + private fun markAccepted(index: Int) { + // Persist the flag first, atomically against concurrent flips and + // reconciles. This is best-effort. A lost flag only re-sends this part on + // a later resume, and the consumer's accept rules absorb that ('already + // completed'). That is better than a failure of an upload that the server + // accepted. + manifest = store.update(uploadId) { it.withPartAccepted(index) } + ?: manifest?.withPartAccepted(index) + manifest?.parts?.get(index)?.let { acceptedBytes.addAndGet(it.size) } + partSent.remove(index) + reportProgress() + } + + private fun onPartProgress(index: Int, sent: Long) { + if (sent == 0L) partSent.remove(index) else partSent[index] = sent + reportProgress() + } + + private fun reportProgress() { + val total = manifest?.totalBytes ?: return + val sent = (acceptedBytes.get() + partSent.values.sum()).coerceAtMost(total) + UploadProgress.set(uploadId, sent) + EventReporter.progress(uploadId, sent, total) + updateNotification() + } + + // A resume of a finished-but-unacknowledged upload (all parts accepted, + // 'completed' journaled, and the consumer re-called startUpload before the + // ack) must not mint a second terminal event. Re-emit the journaled one. + // Then a live listener still hears it, with the eventId that the consumer + // will acknowledge. And a trailing run whose completion was already ACKED + // reports nothing at all. See ChunkedEngine.CompletionReport. + private fun journalCompleted(freshCompletion: Boolean) { + val report = ChunkedEngine.completionReport( + EventJournal.get(context).unacknowledged(), + uploadId, + freshCompletion, + ) + when (report) { + is ChunkedEngine.CompletionReport.ReEmit -> EventReporter.emit(report.entry) + // No response fields, because no single response represents N accepted + // parts. + ChunkedEngine.CompletionReport.Mint -> journalAndEmit( + EventJournal.Entry( + eventId = UUID.randomUUID().toString(), + uploadId = uploadId, + type = "completed", + timestamp = System.currentTimeMillis(), + ), + ) + ChunkedEngine.CompletionReport.None -> Unit + } + } + + private fun handleFailure(error: Throwable) { + val entry = when (error) { + is ExpiredException -> errorEntry( + error = "upload expired before every part was accepted", + errorKind = "expired", + ) + is PartRejectedException -> { + val (body, truncated) = EventJournal.capBody(error.response.body) + errorEntry( + error = "HTTP ${error.response.code} on part ${error.partIndex}", + errorKind = "http", + partIndex = error.partIndex, + responseCode = error.response.code, + responseBody = body, + responseBodyTruncated = truncated, + responseHeaders = error.response.headers, + ) + } + is SourceMissingException -> errorEntry(error = error.message!!, errorKind = "file") + is PartBeyondEofException -> errorEntry( + error = error.message!!, + errorKind = "file", + partIndex = error.partIndex, + ) + else -> { + val fileExists = manifest?.let { m -> + runCatching { File(m.sourcePath).exists() }.getOrDefault(true) + } ?: true + errorEntry( + error = error.message ?: "Unknown exception", + errorKind = UploadOutcome.errorKind(error, fileExists), + ) + } + } + journalAndEmit(entry) + } + + private fun errorEntry( + error: String, + errorKind: String, + partIndex: Int? = null, + responseCode: Int? = null, + responseBody: String? = null, + responseBodyTruncated: Boolean = false, + responseHeaders: Map? = null, + ) = EventJournal.Entry( + eventId = UUID.randomUUID().toString(), + uploadId = uploadId, + type = "error", + timestamp = System.currentTimeMillis(), + error = error, + errorKind = errorKind, + partIndex = partIndex, + responseCode = responseCode, + responseBody = responseBody, + responseBodyTruncated = responseBodyTruncated, + responseHeaders = responseHeaders, + ) + + // The semantics are the same as UploadWorker's. Only a user cancel is + // terminal (journaled, cancelReason 'user'). A system stop emits nothing, + // because WorkManager will re-run this upload, and the manifest resumes it. + // The manifest and the bytes are kept in both cases. stopUpload's contract + // is that the next startUpload resumes. + private fun checkAndHandleCancellation(): Boolean { + if (!isStopped) return false + + UploadProgress.remove(uploadId) + + if (!UserCancellations.consume(uploadId)) return true + + journalAndEmit( + EventJournal.Entry( + eventId = UUID.randomUUID().toString(), + uploadId = uploadId, + type = "cancelled", + timestamp = System.currentTimeMillis(), + cancelReason = "user", + ), + ) + return true + } + + private fun journalAndEmit(entry: EventJournal.Entry) { + // A terminal event for an id whose manifest is gone would report an + // upload that nobody owns any more. Either removeUpload deleted it mid-run + // (its work cancel races the in-flight PUT's IOException), or a completed + // ack released it. Suppress the event; iOS's removedIds has the same idea. + // A user cancel keeps its manifest, so real 'cancelled' events pass + // through. + if (!store.contains(uploadId)) return + EventReporter.journalAndEmit(context, entry) + } + + private fun validateAndReportConnectivity(wifiOnly: Boolean): Boolean { + connectivity = validateConnectivity(context, wifiOnly) + updateNotification() + return connectivity == Connectivity.Ok + } + + private fun updateNotification() { + if (manifest?.showsNotification != true) return + notificationManager.notify( + config.systemNotificationId, + buildUploadNotification(context, config, connectivity), + ) + } + + override suspend fun getForegroundInfo(): ForegroundInfo = + uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) +} + +/** + * The Result that a chunked run returns after it journals a terminal error: + * SUCCESS, deliberately. The journal and the manifest are the upload's outcome + * record, never the WorkManager row state. A row that finishes FAILED destroys + * every appended dependent: WorkManager marks the dependents of a failed + * prerequisite FAILED without a run. Thus a resume enqueued during the failing + * run's teardown window would silently never run (see the APPEND_OR_REPLACE + * note in UploaderModule.enqueueChunkedUpload). getAllUploads derives a + * chunked upload's state from its manifest (allAccepted), not from row states. + */ +internal fun terminalErrorResult(): ListenableWorker.Result = ListenableWorker.Result.success() diff --git a/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt b/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt new file mode 100644 index 00000000..424c627b --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/ChunkedWorkerGate.kt @@ -0,0 +1,40 @@ +package ai.openspace.backgroundupload + +import java.util.concurrent.ConcurrentHashMap + +/** + * At most one [ChunkedUploadWorker] EXECUTES per upload id, process-wide. + * + * The unique-work chain almost guarantees this, but not across a cancel. + * cancelUniqueWork marks the row CANCELLED immediately, while the cancelled + * worker's coroutine still winds down. Thus a startUpload that arrives right + * after a cancelUpload can enqueue (and start) a replacement worker while the + * old worker still has a part PUT in flight. Two concurrent PUTs of one + * partNum are verified unsafe on the server side. A starting worker acquires + * its id here, and a successor waits for the release. + * + * This is also the truthful "is this upload running" for the recreate rule. + * A worker registers before its first manifest read, and it releases in a + * finally block. WorkManager's row state stays RUNNING for a moment after + * doWork returns. This gate does not: it never reports a finished run as + * running. + * + * The gate is same-process only, like [UserCancellations]. A worker in a dead + * process holds nothing, and WorkManager runs our workers in the app process. + */ +object ChunkedWorkerGate { + private val holders = ConcurrentHashMap() + + /** True when [token] now holds the id, or already held it. False while another token holds it. */ + fun tryAcquire(id: String, token: Any): Boolean { + val current = holders.putIfAbsent(id, token) + return current == null || current === token + } + + /** Releases only when [token] is the holder. Thus a stale release cannot evict a successor. */ + fun release(id: String, token: Any) { + holders.remove(id, token) + } + + fun isRunning(id: String): Boolean = holders.containsKey(id) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt index 5bbc0738..2dd718b3 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventJournal.kt @@ -29,8 +29,11 @@ class EventJournal( val responseBodyTruncated: Boolean = false, val responseHeaders: Map? = null, val error: String? = null, - val errorKind: String? = null, // http | network | file | unknown + val errorKind: String? = null, // http | network | file | expired | unknown val cancelReason: String? = null, // user | system + // Chunked uploads: the index of the failing part, when one part's response + // caused the error. + val partIndex: Int? = null, ) { fun toWritableMap(): com.facebook.react.bridge.WritableMap = com.facebook.react.bridge.Arguments.createMap().apply { @@ -47,6 +50,7 @@ class EventJournal( error?.let { putString("error", it) } errorKind?.let { putString("errorKind", it) } cancelReason?.let { putString("cancelReason", it) } + partIndex?.let { putInt("partIndex", it) } } } diff --git a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt index a4770c78..46460e1e 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/EventReporter.kt @@ -1,5 +1,6 @@ package ai.openspace.backgroundupload +import android.content.Context import com.facebook.react.bridge.Arguments // Sends live events to JS through the module's codegen event emitters. Terminal @@ -8,6 +9,14 @@ import com.facebook.react.bridge.Arguments // it up from getUnacknowledgedEvents instead. object EventReporter { + // Journal first, then emit. The journal is the durable record; it survives + // when JS is dead. The live emit is best-effort. The two carry the identical + // payload. Thus a consumer can acknowledge a live event by its eventId. + fun journalAndEmit(context: Context, entry: EventJournal.Entry) { + EventJournal.get(context).append(entry) + emit(entry) + } + // Emit a terminal event from its journal entry, so the live event carries the // exact same payload (incl. eventId) as the journaled copy — letting a consumer // ackEvents([eventId]) right after handling a live event, and keeping iOS/Android diff --git a/android/src/main/java/ai/openspace/backgroundupload/Upload.kt b/android/src/main/java/ai/openspace/backgroundupload/Upload.kt index dc556693..3df469b0 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/Upload.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/Upload.kt @@ -1,5 +1,6 @@ package ai.openspace.backgroundupload +import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import java.util.UUID @@ -12,10 +13,10 @@ data class Upload( val path: String, val method: String, val wifiOnly: Boolean, - // Non-2xx statuses to treat as a successful completion (e.g. [409] when - // duplicate-create conflicts are expected). Everything else non-2xx is a - // terminal http error. Empty by default. - val acceptStatus: List, + // Non-2xx responses to treat as a successful completion (for example, a 409 + // whose body marks an expected duplicate). Every other non-2xx response is a + // terminal http error. The list is empty by default. + val accept: List, val headers: Map, /** * Suppresses the progress notification for this upload. @@ -32,8 +33,38 @@ data class Upload( */ val noNotification: Boolean, ) { + // v8 persisted `acceptStatus: List` where v9 persists `accept`. This is + // not a constructor parameter. It exists only so Gson can surface the legacy + // field to [normalized]. It is null, and thus never serialized, for every + // upload that this build creates. + private val acceptStatus: List? = null + val showsNotification get() = !noNotification + /** + * Gson does not use the constructor. Thus a WorkManager job that an older + * build enqueued can give this worker an object whose non-null fields are + * null. A v8 job carries `acceptStatus` and no `accept`. That NPEs the first + * time the worker touches [accept], after the file has fully transmitted, + * and the re-runs then re-send the whole file. This is the same + * normalize-after-fromJson pattern as ChunkedManifestStore.validated(): map + * the legacy statuses to rules, default what is absent, and give the worker + * an object that is safe to use. + */ + @Suppress("SENSELESS_COMPARISON", "USELESS_ELVIS") + fun normalized(): Upload = Upload( + id = id, + url = url, + path = path, + method = method ?: "POST", + wifiOnly = wifiOnly, + accept = accept + ?: acceptStatus?.map { UploadOutcome.AcceptRule(it) } + ?: emptyList(), + headers = headers ?: emptyMap(), + noNotification = noNotification, + ) + class MissingOptionException(optionName: String) : IllegalArgumentException("Missing '$optionName'") @@ -44,17 +75,8 @@ data class Upload( path = map.getString(Upload::path.name) ?: throw MissingOptionException(Upload::path.name), method = map.getString(Upload::method.name) ?: "POST", wifiOnly = if (map.hasKey(Upload::wifiOnly.name)) map.getBoolean(Upload::wifiOnly.name) else false, - acceptStatus = map.getArray(Upload::acceptStatus.name)?.let { arr -> - (0 until arr.size()).map { i -> arr.getInt(i) } - } ?: listOf(), - headers = map.getMap(Upload::headers.name).let { headers -> - if (headers == null) return@let mapOf() - val map = mutableMapOf() - for (entry in headers.entryIterator) { - map[entry.key] = entry.value.toString() - } - return@let map - }, + accept = parseAcceptRules(map.getArray(Upload::accept.name)), + headers = parseHeaderMap(map.getMap(Upload::headers.name)), // The notification text and identity are not per-upload options. The // worker reads them from the NotificationConfig that configure() saved. noNotification = if (map.hasKey(Upload::noNotification.name)) @@ -63,5 +85,23 @@ data class Upload( } } +// Upload and ChunkedManifest share this: one accept-rules shape, one parser. +internal fun parseAcceptRules(arr: ReadableArray?): List { + if (arr == null) return listOf() + return (0 until arr.size()).mapNotNull { i -> + val rule = arr.getMap(i) ?: return@mapNotNull null + UploadOutcome.AcceptRule( + status = rule.getInt("status"), + bodyIncludes = if (rule.hasKey("bodyIncludes")) rule.getString("bodyIncludes") else null, + ) + } +} - +internal fun parseHeaderMap(headers: ReadableMap?): Map { + if (headers == null) return mapOf() + val map = mutableMapOf() + for (entry in headers.entryIterator) { + map[entry.key] = entry.value.toString() + } + return map +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt new file mode 100644 index 00000000..f1b6b90b --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadNotification.kt @@ -0,0 +1,135 @@ +package ai.openspace.backgroundupload + +import android.app.Notification +import android.app.NotificationChannel +import android.app.NotificationManager +import android.app.PendingIntent +import android.content.Context +import android.content.Intent +import android.content.pm.ServiceInfo +import android.net.ConnectivityManager +import android.net.NetworkCapabilities.NET_CAPABILITY_VALIDATED +import android.net.NetworkCapabilities.TRANSPORT_WIFI +import android.os.Build +import android.widget.RemoteViews +import androidx.core.app.NotificationCompat +import androidx.work.ForegroundInfo + +// This file builds the progress notification and probes connectivity. The +// simple and chunked workers share it. All uploads share one notification, +// identified by the NotificationConfig that configure() saved. Its progress +// bar is the total across the uploads. + +internal enum class Connectivity { NoWifi, NoInternet, Ok } + +// This is synchronized to ensure consistent status across workers +@Synchronized +internal fun validateConnectivity(context: Context, wifiOnly: Boolean): Connectivity { + val manager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager + val network = manager.activeNetwork + val capabilities = manager.getNetworkCapabilities(network) + + val hasInternet = capabilities?.hasCapability(NET_CAPABILITY_VALIDATED) == true + + // not wifiOnly, return early + if (!wifiOnly) return if (hasInternet) Connectivity.Ok else Connectivity.NoInternet + + // handle wifiOnly + return if (hasInternet && capabilities?.hasTransport(TRANSPORT_WIFI) == true) + Connectivity.Ok + else + Connectivity.NoWifi // don't return NoInternet here, more direct to request to join wifi +} + +// Makes sure that the channel for the foreground notification exists. It makes +// the channel only when the channel is absent. Thus a channel that the consumer +// registered, with their own name and importance, always wins. When configure() +// never set a channel, we use a default LOW-importance channel, and no notifee +// setup is necessary. +internal fun ensureNotificationChannel(manager: NotificationManager, config: NotificationConfig) { + // minSdk is 29, so NotificationChannel (API 26) is always available. + if (manager.getNotificationChannel(config.notificationChannel) != null) return + val channel = NotificationChannel( + config.notificationChannel, + "Uploads", + NotificationManager.IMPORTANCE_LOW, + ) + manager.createNotificationChannel(channel) +} + +// builds the notification required to enable Foreground mode +internal fun buildUploadNotification( + context: Context, + config: NotificationConfig, + connectivity: Connectivity, +): Notification { + val channel = config.notificationChannel + val progress = UploadProgress.total() + val progress2Decimals = "%.2f".format(progress) + val title = when (connectivity) { + Connectivity.NoWifi -> config.notificationTitleNoWifi + Connectivity.NoInternet -> config.notificationTitleNoInternet + Connectivity.Ok -> config.notificationTitle + } + + // Custom layout for progress notification. + // The default hides the % text. This one shows it on the right, + // like most examples in various docs. + val content = RemoteViews(context.packageName, R.layout.notification) + content.setTextViewText(R.id.notification_title, title) + content.setTextViewText(R.id.notification_progress, "${progress2Decimals}%") + content.setProgressBar(R.id.notification_progress_bar, 100, progress.toInt(), false) + + return NotificationCompat.Builder(context, channel).run { + // Starting Android 12, the notification shows up with a confusing delay of 10s. + // This fixes that delay. + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) + foregroundServiceBehavior = Notification.FOREGROUND_SERVICE_IMMEDIATE + + // Required by android. Here we use the system's default upload icon + setSmallIcon(android.R.drawable.stat_sys_upload) + // These prevent the notification from being force-dismissed or dismissed when pressed + setOngoing(true) + setAutoCancel(false) + // These help show the same custom content when the notification collapses and expands + setCustomContentView(content) + setCustomBigContentView(content) + // opens the app when the notification is pressed + setContentIntent(openAppIntent(context)) + build() + } +} + +internal fun uploadForegroundInfo(config: NotificationConfig, notification: Notification): ForegroundInfo { + val id = config.systemNotificationId + // Starting Android 14, FOREGROUND_SERVICE_TYPE_DATA_SYNC is mandatory, otherwise app will crash + return if (Build.VERSION.SDK_INT > Build.VERSION_CODES.TIRAMISU) + ForegroundInfo(id, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC) + else + ForegroundInfo(id, notification) +} + +/** + * Whether [error] means that the system refused to let this worker enter + * foreground mode, because the app is in the background. Android 12 (API 31) + * restricts foreground-service starts from the background, and WorkManager + * relaunches workers exactly there (a reboot, or a quota resume). The Android + * 15 dataSync time-limit denial surfaces as the same exception. The transfer + * itself needs no foreground mode. It only loses process-priority protection. + * Thus callers continue without it. They do not fail an upload that can run. + * The function walks the causes, because setForeground can wrap the platform + * exception. + */ +internal fun isForegroundStartDenied(error: Throwable): Boolean { + if (Build.VERSION.SDK_INT < 31) return false + // ServiceStartNotAllowedException (API 31) covers the two variants: + // Foreground- and BackgroundServiceStartNotAllowedException. + return generateSequence(error) { it.cause } + .any { it is android.app.ServiceStartNotAllowedException } +} + +private fun openAppIntent(context: Context): PendingIntent? { + val intent = Intent(context, NotificationReceiver::class.java) + val flags = PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE + return PendingIntent.getBroadcast(context, "RNFileUpload-notification".hashCode(), intent, flags) +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt index d044ac9e..f4fb2763 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadOutcome.kt @@ -7,11 +7,26 @@ import java.io.IOException // logic in the uploader (it decides success vs failure), so it's covered directly. object UploadOutcome { - // Whether an HTTP response counts as a successful completion. 2xx always, plus - // any per-request acceptStatus codes (axios validateStatus semantics). Anything - // else — including 4xx/5xx — is a terminal http error, not a completion. - fun isAccepted(code: Int, acceptStatus: List): Boolean = - code in 200..299 || acceptStatus.contains(code) + /** + * A non-2xx response to treat as success. `bodyIncludes` narrows the rule by + * a response-body substring. This is necessary when one status has several + * meanings, and only the message shows the difference (our backend's 409). + * Gson persists it inside [Upload] and [ChunkedManifest]; see + * consumer-rules.pro. + */ + data class AcceptRule( + val status: Int, + val bodyIncludes: String? = null, + ) + + // Whether an HTTP response counts as a successful completion. A 2xx always + // counts, and a matching per-request accept rule counts. Every other response, + // 4xx and 5xx included, is a terminal http error, not a completion. + fun isAccepted(code: Int, body: String?, accept: List): Boolean = + code in 200..299 || accept.any { rule -> + rule.status == code && + (rule.bodyIncludes == null || body?.contains(rule.bodyIncludes) == true) + } // Classify a thrown error into a stable kind for the JS layer. `fileExists` // is passed in (not read here) to keep this pure; callers should default it to diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt new file mode 100644 index 00000000..06718b01 --- /dev/null +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt @@ -0,0 +1,28 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.sync.Semaphore +import okhttp3.OkHttpClient +import java.util.concurrent.TimeUnit + +// The HTTP client and the library-wide transmission gate. The simple and +// chunked workers share them. Thus "requests transmitting at one time" means +// one thing across all uploads. + +// Max total time for a single request to complete +// This is 24hrs so plenty of time for large uploads +// Worst case is the time maxes out and the upload gets restarted. +// Not using unlimited time to prevent unexpected behaviors. +private const val REQUEST_TIMEOUT = 24L +private val REQUEST_TIMEOUT_UNIT = TimeUnit.HOURS + +// The number of requests transmitting at one time across ALL uploads, chunked +// parts included. A semaphore controls this, not OkHttp's connection limits, +// because those limits add a delay between requests. The design's library-wide +// cap is 4. The change from 1 to 4 lands with the hardening slice, not here. +internal const val MAX_TRANSFER_CONCURRENCY = 1 +internal val transferSemaphore = Semaphore(MAX_TRANSFER_CONCURRENCY) + +// Use Okhttp as it provides the most standard behaviors even though it's not coroutine friendly +internal val uploadHttpClient = OkHttpClient.Builder() + .callTimeout(REQUEST_TIMEOUT, REQUEST_TIMEOUT_UNIT) + .build() diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt index 44696541..0e48da98 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadUtils.kt @@ -4,6 +4,7 @@ import kotlinx.coroutines.suspendCancellableCoroutine import okhttp3.Call import okhttp3.Callback import okhttp3.Headers.Companion.toHeaders +import okhttp3.MediaType import okhttp3.OkHttpClient import okhttp3.Request import okhttp3.RequestBody @@ -15,11 +16,14 @@ import okio.ForwardingSink import okio.buffer import java.io.File import java.io.IOException +import java.io.RandomAccessFile import kotlin.coroutines.resumeWithException // Throttling interval of progress reports private const val PROGRESS_INTERVAL = 500 // milliseconds +private const val RANGE_COPY_BUFFER = 64 * 1024 + data class UploadResponse( val code: Int, val body: String, @@ -32,25 +36,36 @@ suspend fun okhttpUpload( upload: Upload, file: File, onProgress: (Long) -> Unit -) = - suspendCancellableCoroutine { continuation -> - val requestBody = file.asRequestBody() - var lastProgressReport = 0L - fun throttled(): Boolean { - val now = System.currentTimeMillis() - if (now - lastProgressReport < PROGRESS_INTERVAL) return true - lastProgressReport = now - return false - } +): UploadResponse { + val request = Request.Builder() + .url(upload.url) + .headers(upload.headers.toHeaders()) + .method(upload.method, withProgressListener(file.asRequestBody(), throttled(onProgress))) + .build() + return awaitResponse(client, request) +} - val request = Request.Builder() - .url(upload.url) - .headers(upload.headers.toHeaders()) - .method(upload.method, withProgressListener(requestBody) { progress -> - if (!throttled()) onProgress(progress) - }) - .build() +/** + * PUTs one byte range of the source file: a chunked part. It streams straight + * from disk, with no temporary chunk file. The headers are the consumer's, + * unchanged. The library adds nothing, per the design's protocol-as-data rule. + */ +suspend fun okhttpUploadPart( + client: OkHttpClient, + part: ChunkedManifest.Part, + file: File, + onProgress: (Long) -> Unit +): UploadResponse { + val request = Request.Builder() + .url(part.url) + .headers(part.headers.toHeaders()) + .put(withProgressListener(rangeRequestBody(file, part.start, part.end), throttled(onProgress))) + .build() + return awaitResponse(client, request) +} +private suspend fun awaitResponse(client: OkHttpClient, request: Request): UploadResponse = + suspendCancellableCoroutine { continuation -> val call = client.newCall(request) continuation.invokeOnCancellation { call.cancel() } call.enqueue(object : Callback { @@ -61,7 +76,11 @@ suspend fun okhttpUpload( val result = response.use { res -> // close the response asap UploadResponse( res.code, - res.body?.string()?.takeIf { str -> str.isNotEmpty() } ?: res.message, + // The body, unchanged: an empty body stays empty. A substituted + // HTTP reason phrase would make accept `bodyIncludes` rules match + // text that the server never sent. iOS also reports the body + // as-is. + res.body?.string().orEmpty(), res.headers.toMultimap().mapValues { it.value.joinToString(", ") } ) } @@ -71,6 +90,47 @@ suspend fun okhttpUpload( }) } +private fun throttled(onProgress: (Long) -> Unit): (Long) -> Unit { + var lastProgressReport = 0L + return { progress -> + val now = System.currentTimeMillis() + if (now - lastProgressReport >= PROGRESS_INTERVAL) { + lastProgressReport = now + onProgress(progress) + } + } +} + +/** + * Streams the file bytes [start, end) as a request body. A RandomAccessFile + * backs it, opened fresh on every writeTo call. OkHttp can replay a body (for + * example, after a connection-level retry), and a one-shot stream would then + * send truncated data silently. + */ +private fun rangeRequestBody(file: File, start: Long, end: Long) = object : RequestBody() { + // Null, so no Content-Type is invented. The consumer's header is already on + // the request, unchanged. + override fun contentType(): MediaType? = null + + override fun contentLength() = end - start + + override fun writeTo(sink: BufferedSink) { + RandomAccessFile(file, "r").use { raf -> + raf.seek(start) + val buffer = ByteArray(RANGE_COPY_BUFFER) + var remaining = end - start + while (remaining > 0L) { + val read = raf.read(buffer, 0, minOf(remaining, buffer.size.toLong()).toInt()) + if (read < 0) throw IOException( + "source file ended before part range [$start, $end): ${file.path}", + ) + sink.write(buffer, 0, read) + remaining -= read + } + } + } +} + // create a request body that allows us to listen to progress. // okhttp has no built-in way of reporting progress private fun withProgressListener( @@ -94,4 +154,4 @@ private fun withProgressListener( body.writeTo(bufferedSink) bufferedSink.flush() } -} \ No newline at end of file +} diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt index e20f1e3c..1f0c63e8 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadWorker.kt @@ -1,36 +1,20 @@ package ai.openspace.backgroundupload -import android.app.Notification -import android.app.NotificationChannel import android.app.NotificationManager -import android.app.PendingIntent import android.content.Context -import android.content.Intent -import android.content.pm.ServiceInfo -import android.net.ConnectivityManager -import android.net.NetworkCapabilities.NET_CAPABILITY_VALIDATED -import android.net.NetworkCapabilities.TRANSPORT_WIFI -import android.os.Build -import android.widget.RemoteViews -import androidx.core.app.NotificationCompat import androidx.work.CoroutineWorker import androidx.work.ForegroundInfo import androidx.work.WorkerParameters import com.google.gson.Gson import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.delay -import kotlinx.coroutines.sync.Semaphore import kotlinx.coroutines.withContext -import okhttp3.OkHttpClient import java.io.File import java.io.IOException import java.net.UnknownHostException import java.util.UUID import java.util.concurrent.TimeUnit -// All workers will start `doWork` immediately but only 1 request is active at a time. -private const val MAX_CONCURRENCY = 1 - // Retry delay private val RETRY_DELAY = TimeUnit.SECONDS.toMillis(10L) @@ -39,24 +23,6 @@ private val RETRY_DELAY = TimeUnit.SECONDS.toMillis(10L) // library. It is not an option. private const val MAX_RETRIES = 5 -// Max total time for a single request to complete -// This is 24hrs so plenty of time for large uploads -// Worst case is the time maxes out and the upload gets restarted. -// Not using unlimited time to prevent unexpected behaviors. -private const val REQUEST_TIMEOUT = 24L -private val REQUEST_TIMEOUT_UNIT = TimeUnit.HOURS - -// Control max concurrent requests using semaphore to instead of using -// `maxConnectionsCount` in HttpClient as the latter introduces a delay between requests -private val semaphore = Semaphore(MAX_CONCURRENCY) - -// Use Okhttp as it provides the most standard behaviors even though it's not coroutine friendly -private val client = OkHttpClient.Builder() - .callTimeout(REQUEST_TIMEOUT, REQUEST_TIMEOUT_UNIT) - .build() - -private enum class Connectivity { NoWifi, NoInternet, Ok } - class UploadWorker(private val context: Context, params: WorkerParameters) : CoroutineWorker(context, params) { @@ -88,7 +54,10 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : // However, the only way it has errors is the implementation is incorrect, // which can be caught in development val paramsJson = inputData.getString(PARAMS_KEY) ?: throw Throwable("No Params") - upload = Gson().fromJson(paramsJson, Upload::class.java) + // normalized(): an older build can have enqueued this job, and its JSON + // shape can make non-null fields null (Gson does not use the constructor). + // See Upload.normalized. + upload = Gson().fromJson(paramsJson, Upload::class.java).normalized() // initialization, errors thrown here won't be retried try { @@ -97,7 +66,7 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : if (upload.showsNotification) { // The foreground notification needs a channel to exist first, or posting // it silently fails and setForeground can crash on newer Android. - ensureNotificationChannel() + ensureNotificationChannel(notificationManager, config) // `setForeground` is recommended for long-running workers. // Foreground mode helps prioritize the worker, reducing the risk // of it being killed during low memory or Doze/App Standby situations. @@ -105,8 +74,13 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : setForeground(getForegroundInfo()) } } catch (error: Throwable) { - if (!checkAndHandleCancellation()) handleError(error) - throw error + if (!isForegroundStartDenied(error)) { + if (!checkAndHandleCancellation()) handleError(error) + throw error + } + // The app is in the background on API 31+ (see isForegroundStartDenied). + // Continue the upload without foreground priority. Do not fail an upload + // that can run. } @@ -122,11 +96,12 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : // which cancels the delay immediately and throws CancellationException. // - Linear backoff instead of exponential. One reason for this is we retry on // invalid connections. Exponential will take too long. - // - We only retry transport failures here (no response). Any HTTP response, - // including 4xx/5xx, is terminal at this layer: handleResponse classifies it - // (2xx/acceptStatus -> completed, else http error) and the worker returns - // without retrying. Response-code-based retry policy is the JS queue's job. - // This is consistent with iOS behavior. + // - We retry only transport failures here (no response). An HTTP + // response, 4xx and 5xx included, is terminal at this layer. + // handleResponse classifies it (a 2xx or an accept rule -> completed, + // else an http error), and the worker returns without a retry. A + // response-code retry policy is the JS queue's job. This matches the + // iOS behavior. if (isRetried) delay(RETRY_DELAY) isRetried = true @@ -157,10 +132,10 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : if (!validateAndReportConnectivity()) return null // wait for its turn to run - semaphore.acquire() + transferSemaphore.acquire() try { - return okhttpUpload(client, upload, file) { progress -> + return okhttpUpload(uploadHttpClient, upload, file) { progress -> handleProgress(progress, size) } } catch (error: Throwable) { @@ -169,7 +144,7 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : // pass the error to upper layer for retry decision throw error } finally { - semaphore.release() + transferSemaphore.release() } } @@ -183,16 +158,20 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : // worker never posted one, and `notify` would create it outside foreground mode. private fun updateNotification() { if (!upload.showsNotification) return - notificationManager.notify(config.systemNotificationId, buildNotification()) + notificationManager.notify( + config.systemNotificationId, + buildUploadNotification(context, config, connectivity), + ) } - // An HTTP response came back. "completed" only for 2xx or a per-request - // acceptStatus code (axios validateStatus semantics — a 400 is an error, not a - // completion); anything else is a terminal http error carrying the full - // response. Either way the request finished, so the worker does not retry. + // An HTTP response came back. It is "completed" only for a 2xx or a matching + // accept rule (axios validateStatus semantics: a 400 is an error, not a + // completion). Every other response is a terminal http error that carries the + // full response. In both cases the request finished, so the worker does not + // retry. private fun handleResponse(response: UploadResponse) { UploadProgress.complete(upload.id) - val accepted = UploadOutcome.isAccepted(response.code, upload.acceptStatus) + val accepted = UploadOutcome.isAccepted(response.code, response.body, upload.accept) val (body, truncated) = EventJournal.capBody(response.body) journalAndEmit( EventJournal.Entry( @@ -261,13 +240,8 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : return true } - // Journal before emitting: the journal is the durable record (survives JS being - // dead); the live emit is best-effort. Both carry the identical payload, so a - // consumer can ack a live event by its eventId. - private fun journalAndEmit(entry: EventJournal.Entry) { - EventJournal.get(context).append(entry) - EventReporter.emit(entry) - } + private fun journalAndEmit(entry: EventJournal.Entry) = + EventReporter.journalAndEmit(context, entry) /** @return whether to retry */ private fun checkRetry(error: Throwable): Boolean { @@ -307,94 +281,6 @@ class UploadWorker(private val context: Context, params: WorkerParameters) : return this.connectivity == Connectivity.Ok } - // Makes sure that the channel for the foreground notification exists. It - // makes the channel only when the channel is absent. Thus a channel that the - // consumer registered, with their own name and importance, always wins. When - // configure() never set a channel, we use a default LOW-importance channel, - // and no notifee setup is necessary. - private fun ensureNotificationChannel() { - // minSdk is 29, so NotificationChannel (API 26) is always available. - if (notificationManager.getNotificationChannel(config.notificationChannel) != null) return - val channel = NotificationChannel( - config.notificationChannel, - "Uploads", - NotificationManager.IMPORTANCE_LOW, - ) - notificationManager.createNotificationChannel(channel) - } - - // builds the notification required to enable Foreground mode - fun buildNotification(): Notification { - val channel = config.notificationChannel - val progress = UploadProgress.total() - val progress2Decimals = "%.2f".format(progress) - val title = when (connectivity) { - Connectivity.NoWifi -> config.notificationTitleNoWifi - Connectivity.NoInternet -> config.notificationTitleNoInternet - Connectivity.Ok -> config.notificationTitle - } - - // Custom layout for progress notification. - // The default hides the % text. This one shows it on the right, - // like most examples in various docs. - val content = RemoteViews(context.packageName, R.layout.notification) - content.setTextViewText(R.id.notification_title, title) - content.setTextViewText(R.id.notification_progress, "${progress2Decimals}%") - content.setProgressBar(R.id.notification_progress_bar, 100, progress.toInt(), false) - - return NotificationCompat.Builder(context, channel).run { - // Starting Android 12, the notification shows up with a confusing delay of 10s. - // This fixes that delay. - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.S) - foregroundServiceBehavior = Notification.FOREGROUND_SERVICE_IMMEDIATE - - // Required by android. Here we use the system's default upload icon - setSmallIcon(android.R.drawable.stat_sys_upload) - // These prevent the notification from being force-dismissed or dismissed when pressed - setOngoing(true) - setAutoCancel(false) - // These help show the same custom content when the notification collapses and expands - setCustomContentView(content) - setCustomBigContentView(content) - // opens the app when the notification is pressed - setContentIntent(openAppIntent(context)) - build() - } - } - - override suspend fun getForegroundInfo(): ForegroundInfo { - val notification = buildNotification() - val id = config.systemNotificationId - // Starting Android 14, FOREGROUND_SERVICE_TYPE_DATA_SYNC is mandatory, otherwise app will crash - return if (Build.VERSION.SDK_INT > Build.VERSION_CODES.TIRAMISU) - ForegroundInfo(id, notification, ServiceInfo.FOREGROUND_SERVICE_TYPE_DATA_SYNC) - else - ForegroundInfo(id, notification) - } -} - -// This is outside and synchronized to ensure consistent status across workers -@Synchronized -private fun validateConnectivity(context: Context, wifiOnly: Boolean): Connectivity { - val manager = context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager - val network = manager.activeNetwork - val capabilities = manager.getNetworkCapabilities(network) - - val hasInternet = capabilities?.hasCapability(NET_CAPABILITY_VALIDATED) == true - - // not wifiOnly, return early - if (!wifiOnly) return if (hasInternet) Connectivity.Ok else Connectivity.NoInternet - - // handle wifiOnly - return if (hasInternet && capabilities?.hasTransport(TRANSPORT_WIFI) == true) - Connectivity.Ok - else - Connectivity.NoWifi // don't return NoInternet here, more direct to request to join wifi -} - - -private fun openAppIntent(context: Context): PendingIntent? { - val intent = Intent(context, NotificationReceiver::class.java) - val flags = PendingIntent.FLAG_UPDATE_CURRENT or PendingIntent.FLAG_IMMUTABLE - return PendingIntent.getBroadcast(context, "RNFileUpload-notification".hashCode(), intent, flags) + override suspend fun getForegroundInfo(): ForegroundInfo = + uploadForegroundInfo(config, buildUploadNotification(context, config, connectivity)) } diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt index 209c89d6..206253d7 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploaderModule.kt @@ -13,6 +13,9 @@ import com.facebook.react.bridge.ReadableArray import com.facebook.react.bridge.ReadableMap import com.facebook.react.bridge.WritableMap import com.google.gson.Gson +import java.io.File +import java.nio.file.Files +import java.nio.file.StandardCopyOption import java.util.UUID @@ -114,7 +117,19 @@ class UploaderModule(context: ReactApplicationContext) : override fun ackEvents(ids: ReadableArray, promise: Promise) { try { val eventIds = (0 until ids.size()).mapNotNull { ids.getString(it) } - EventJournal.get(reactApplicationContext).ack(eventIds) + val journal = EventJournal.get(reactApplicationContext) + // An acknowledged 'completed' is the ONE moment when a chunked upload's + // manifest and moved bytes may be deleted. Every other terminal keeps + // them for a resume. Resolve which uploads those are before the entries + // are removed. + val completedUploadIds = journal.unacknowledged() + .filter { it.type == "completed" && eventIds.contains(it.eventId) } + .map { it.uploadId } + journal.ack(eventIds) + releaseAckedCompletions( + completedUploadIds, + ChunkedManifestStore.get(reactApplicationContext), + ) { id -> workManager.cancelUniqueWork(id) } promise.resolve(true) } catch (exc: Throwable) { Log.e(TAG, exc.message, exc) @@ -124,30 +139,51 @@ class UploaderModule(context: ReactApplicationContext) : /** - * Enumerates uploads WorkManager still knows about, as [{ id, state }]. - * WorkManager auto-prunes finished work after roughly a day, so this is for - * reconciling live/recent uploads — terminal outcomes must be read from - * getUnacknowledgedEvents, which is durable until acknowledged. + * Enumerates the uploads that WorkManager still knows about, as + * [{ id, state }]. Chunked uploads also carry the aggregate + * { bytesSent, totalBytes }, and they are listed from their durable + * manifests, even after WorkManager prunes finished work (in roughly a day). + * Terminal outcomes must be read from getUnacknowledgedEvents, which is + * durable until acknowledged. */ override fun getAllUploads(promise: Promise) { try { - val infos = workManager.getWorkInfosByTag(WORKER_TAG).get() + val manifests = ChunkedManifestStore.get(reactApplicationContext).all() + .associateBy { it.id } + // Several WorkInfo rows can exist for one upload id. Finished chains + // linger until they are pruned (in roughly a day), and APPEND_OR_REPLACE + // resumes add rows. Thus the rows are grouped, and each id gets exactly + // ONE entry, like iOS (one row per upload, with the same state + // vocabulary). + val statesById = workManager.getWorkInfosByTag(WORKER_TAG).get() + .groupBy( + { info -> + info.tags.firstOrNull { it.startsWith(ID_TAG_PREFIX) } + ?.removePrefix(ID_TAG_PREFIX) + }, + { it.state }, + ) val arr = Arguments.createArray() - for (info in infos) { - val id = info.tags.firstOrNull { it.startsWith(ID_TAG_PREFIX) } - ?.removePrefix(ID_TAG_PREFIX) ?: continue + for ((id, states) in statesById) { + if (id == null || id in manifests) continue arr.pushMap(Arguments.createMap().apply { putString("id", id) + putString("state", simpleUploadState(states)) + }) + } + // Chunked uploads are listed from their durable manifests, which outlive + // the WorkManager rows. Only a live row contributes (running or pending). + // A lingering finished row never speaks for a manifest that is really + // "finished, awaiting its ack" or "stalled, awaiting a startUpload + // resume". + for (manifest in manifests.values) { + arr.pushMap(Arguments.createMap().apply { + putString("id", manifest.id) putString( "state", - when (info.state) { - WorkInfo.State.ENQUEUED, WorkInfo.State.BLOCKED -> "pending" - WorkInfo.State.RUNNING -> "running" - WorkInfo.State.SUCCEEDED -> "completed" - WorkInfo.State.FAILED -> "error" - WorkInfo.State.CANCELLED -> "cancelled" - }, + chunkedUploadState(statesById[manifest.id].orEmpty(), manifest.allAccepted), ) + putChunkedBytes(manifest) }) } promise.resolve(arr) @@ -217,6 +253,138 @@ class UploaderModule(context: ReactApplicationContext) : } + /** + * Starts, or resumes, a chunked upload. It is idempotent against the durable + * [ChunkedManifest]. A first call takes ownership of the source file (an + * O(1) rename into the library's directory) and persists the manifest. A + * re-call with the same id reconciles instead: identical parts are required, + * the stored headers are replaced, and the accepted parts are skipped. Crash + * recovery, a resume after a stop, and a resume with fresh auth are all this + * same call. + */ + override fun startChunkedUpload(options: ReadableMap, promise: Promise) { + try { + promise.resolve(enqueueChunkedUpload(options)) + } catch (exc: Throwable) { + if (exc !is IllegalArgumentException) { + exc.printStackTrace() + Log.e(TAG, exc.message, exc) + } + promise.reject(exc) + } + } + + private fun enqueueChunkedUpload(options: ReadableMap): String { + val store = ChunkedManifestStore.get(reactApplicationContext) + val id = options.getString("id") + ?: throw Upload.MissingOptionException("id") + val blob = store.blobFile(id) + val incoming = ChunkedManifest.fromReadableMap( + options, + sourcePath = blob.absolutePath, + createdAt = System.currentTimeMillis(), + ) + + // One atomic store operation, persisted BEFORE the work is enqueued. The + // manifest is what a worker relaunched with no JS runs from. The store + // lock spans load, reconcile, and save. Thus a running worker's + // markAccepted can never land between them and be erased. The running flag + // inside the lock is race-free too. A worker acquires ChunkedWorkerGate + // before its first manifest read. Thus it either registers first (and the + // recreate is rejected), or it reads the manifest that this call saved. + store.compute(id) { existing -> + if (existing == null) { + val path = options.getString("path") ?: throw Upload.MissingOptionException("path") + takeOwnership(File(path), blob) + incoming + } else { + // `path` is deliberately ignored here. When a manifest exists, the + // owned bytes are the source of truth. + existing.reconcile( + incoming, + running = ChunkedWorkerGate.isRunning(id), + blobSize = File(existing.sourcePath).length(), + ) + } + } + + // The stale-mark reasoning is the same as in enqueueUpload. + UserCancellations.consume(id) + + // A queued successor (an unfinished row that is not RUNNING) already + // guarantees a run after the current one finishes. An appended second run + // would only stack duplicate no-op runs. The manifest reconcile above + // still landed. That is how this call's fresh headers reach the queued + // run. + val states = workManager.getWorkInfosForUniqueWork(id).get().map { it.state } + if (hasQueuedSuccessor(states)) return id + + val request = OneTimeWorkRequestBuilder() + .addTag(WORKER_TAG) + .addTag(ID_TAG_PREFIX + id) + .setInputData(workDataOf(ChunkedUploadWorker.ID_KEY to id)) + .build() + + // APPEND_OR_REPLACE, not KEEP. A worker journals its terminal error before + // doWork returns. Thus a consumer that resumes from the error handler can + // arrive while that run's row is still RUNNING. KEEP would silently drop + // the resume, and nothing would ever run it. An append keeps the runs + // strictly sequential, and a trailing run over an already-settled manifest + // is a clean no-op (see ChunkedEngine.startAction). Appended work is a + // chain DEPENDENT: WorkManager marks the dependents of a failed + // prerequisite FAILED without a run. That is why ChunkedUploadWorker + // always returns Result.success(), even after it journals a terminal error + // (see terminalErrorResult). The OR_REPLACE half only rescues enqueues + // that arrive AFTER the chain already settled failed or cancelled: it + // starts a fresh sequence. A re-call while the worker runs still never + // restarts it. The running worker re-reads the stored manifest before + // every part attempt, so a resume's fresh headers reach it. + workManager + .beginUniqueWork(id, ExistingWorkPolicy.APPEND_OR_REPLACE, request) + .enqueue() + + return id + } + + private fun takeOwnership(source: File, blob: File) { + if (!source.exists()) { + // A crash between the rename and the manifest save leaves the bytes at + // the blob path with no manifest. Adopt them. Do not fail the retry. + if (blob.exists()) return + throw IllegalArgumentException("chunked source file does not exist: ${source.path}") + } + blob.parentFile?.mkdirs() + if (blob.exists()) blob.delete() + if (source.renameTo(blob)) return + // renameTo cannot cross filesystems. Files.move falls back to copy+delete. + Files.move(source.toPath(), blob.toPath(), StandardCopyOption.REPLACE_EXISTING) + } + + + /** + * Releases an upload's stored state. It cancels the scheduled or running + * work, then deletes the chunked manifest and the moved bytes. It is safe on + * any id. A simple upload has nothing stored, so the call reduces to the + * work cancel. There is deliberately no 'cancelled' event. This is an + * explicit release by the consumer, not an outcome that the consumer awaits. + */ + override fun removeUpload(id: String, promise: Promise) { + try { + workManager.cancelUniqueWork(id) + // No user-cancel mark was set, so a running worker's stop handler + // reports nothing. The consume call clears a stale mark from a prior + // life. + UserCancellations.consume(id) + ChunkedManifestStore.get(reactApplicationContext).remove(id) + promise.resolve(null) + } catch (exc: Throwable) { + exc.printStackTrace() + Log.e(TAG, exc.message, exc) + promise.reject(exc) + } + } + + /* * Cancels file upload * Accepts upload ID as a first argument, this upload will be cancelled @@ -224,10 +392,11 @@ class UploaderModule(context: ReactApplicationContext) : */ override fun cancelUpload(id: String, promise: Promise) { try { - val active = workManager.getWorkInfosForUniqueWork(id).get() - .firstOrNull { !it.state.isFinished } + val activeStates = workManager.getWorkInfosForUniqueWork(id).get() + .map { it.state } + .filter { !it.isFinished } - if (active == null) { + if (activeStates.isEmpty()) { // Nothing to cancel. Drop any mark so a later upload reusing this id // can't be misreported as a user cancel. UserCancellations.consume(id) @@ -236,26 +405,29 @@ class UploaderModule(context: ReactApplicationContext) : return } - // Record intent BEFORE cancelling so the worker's stop handler can tell - // this apart from a system stop and report cancelReason 'user'. + // Record the intent BEFORE the cancel. Then a running worker's stop + // handler can tell this apart from a system stop, and it reports + // cancelReason 'user'. UserCancellations.mark(id) workManager.cancelUniqueWork(id) - if (active.state == WorkInfo.State.ENQUEUED) { - // The worker never started, so it will never run its own stop handler and - // nothing else would ever report this cancellation — leaving a consumer - // awaiting this upload's outcome forever. Report it here instead, and - // consume the mark so it cannot leak. + if (cancelReportsFromModule(activeStates)) { + // No worker ever started: the rows are only ENQUEUED, or BLOCKED + // behind an appended chain. Thus no stop handler will ever run, and + // nothing else would ever report this cancellation. A consumer would + // then await this upload's outcome forever. Report it here instead, + // and consume the mark so it cannot leak. UserCancellations.consume(id) - val entry = EventJournal.Entry( - eventId = UUID.randomUUID().toString(), - uploadId = id, - type = "cancelled", - timestamp = System.currentTimeMillis(), - cancelReason = "user", + EventReporter.journalAndEmit( + reactApplicationContext, + EventJournal.Entry( + eventId = UUID.randomUUID().toString(), + uploadId = id, + type = "cancelled", + timestamp = System.currentTimeMillis(), + cancelReason = "user", + ), ) - EventJournal.get(reactApplicationContext).append(entry) - EventReporter.emit(entry) } promise.resolve(true) @@ -265,14 +437,82 @@ class UploaderModule(context: ReactApplicationContext) : promise.reject(exc) } } +} + +/** + * Releases the uploads whose 'completed' events were just acknowledged. That + * is the ONE moment when a chunked upload's manifest and moved bytes may be + * deleted. The allAccepted guard protects a recreate: the id may have been + * RECREATED (a different parts array under the same id) and run again over + * these bytes. An ack of the old life's completion must not cancel that work, + * and it must not delete the blob under it. Simple uploads have no manifest + * and fall through untouched. A cancel of their unique work could kill an + * unrelated new upload that reuses the id. + */ +internal fun releaseAckedCompletions( + uploadIds: List, + store: ChunkedManifestStore, + cancelWork: (String) -> Unit, +) { + uploadIds.forEach { id -> + val manifest = store.load(id) ?: return@forEach + if (!manifest.allAccepted) return@forEach + // Cancel a still-enqueued trailing run BEFORE the delete. A worker that + // starts after the delete finds nothing. It exits silently, but there is + // no reason to run it at all. + cancelWork(id) + store.remove(id) + } +} - // The chunked transport arrives in the platform changes that follow this - // spec change. Until then, a rejection satisfies the codegen contract. - override fun startChunkedUpload(options: ReadableMap, promise: Promise) { - promise.reject("E_NOT_IMPLEMENTED", "Chunked uploads are not implemented in this build") +/** + * Whether cancelUpload must journal and emit the 'cancelled' event itself. + * That is the case only when NO row is RUNNING. A never-started row (ENQUEUED, + * or BLOCKED as an appended chain's dependent) has no worker to run a stop + * handler. A RUNNING worker's stop handler owns the report, including a worker + * that still waits on the ChunkedWorkerGate. + */ +internal fun cancelReportsFromModule(unfinishedStates: List): Boolean = + unfinishedStates.isNotEmpty() && unfinishedStates.none { it == WorkInfo.State.RUNNING } + +/** An unfinished row that is not RUNNING: a queued run that did not start yet. */ +internal fun hasQueuedSuccessor(states: List): Boolean = + states.any { !it.isFinished && it != WorkInfo.State.RUNNING } + +// The aggregate byte fields for a chunked upload's snapshot. bytesSent counts +// accepted parts only. That is the durable number, and it has meaning even in +// a process where the worker does not run. +private fun WritableMap.putChunkedBytes(manifest: ChunkedManifest) { + putDouble("bytesSent", manifest.acceptedBytes.toDouble()) + putDouble("totalBytes", manifest.totalBytes.toDouble()) +} + +/** + * One state for a chunked upload id, from all its WorkInfo rows plus the + * durable manifest. A live row wins. With no live row, the manifest speaks. + * The state is never "cancelled". iOS getAllUploads has no lingering cancelled + * rows (a cancelled task leaves the session). And on Android, a cancelled + * chunked upload keeps its manifest. Its truthful state is + * stalled-awaiting-resume, that is, "error". + */ +internal fun chunkedUploadState(states: List, allAccepted: Boolean): String = + when { + WorkInfo.State.RUNNING in states -> "running" + states.any { !it.isFinished } -> "pending" + allAccepted -> "completed" + else -> "error" } - override fun removeUpload(id: String, promise: Promise) { - promise.reject("E_NOT_IMPLEMENTED", "removeUpload is not implemented in this build") +/** + * One state for a simple upload id, from all its WorkInfo rows. An id can have + * a lingering finished chain next to a live one. A live row wins. Otherwise + * the most conclusive finished state wins. + */ +internal fun simpleUploadState(states: List): String = + when { + WorkInfo.State.RUNNING in states -> "running" + states.any { !it.isFinished } -> "pending" + WorkInfo.State.SUCCEEDED in states -> "completed" + WorkInfo.State.FAILED in states -> "error" + else -> "cancelled" } -} diff --git a/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt b/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt new file mode 100644 index 00000000..0afe1e88 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/AckReleaseTest.kt @@ -0,0 +1,69 @@ +package ai.openspace.backgroundupload + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNotNull +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder + +// An acknowledged 'completed' is the one moment when a chunked upload's stored +// state may be released. But only the completed life's state may go. A recreate +// under the same id can run over the same bytes, and it must survive the old +// life's ack. +class AckReleaseTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun manifest(id: String, accepted: Boolean) = ChunkedManifest( + id = id, + sourcePath = "/data/blob", + parts = listOf( + ChunkedManifest.Part( + url = "https://example.com/1", + headers = emptyMap(), + start = 0, + end = 100, + accepted = accepted, + ), + ), + accept = emptyList(), + expiresAt = 5_000, + wifiOnly = false, + noNotification = false, + createdAt = 1_000, + ) + + @Test + fun `releases a completed upload's manifest and cancels its trailing runs`() { + val store = ChunkedManifestStore(tmp.newFolder()) + store.save(manifest("u1", accepted = true)) + val cancelled = mutableListOf() + releaseAckedCompletions(listOf("u1"), store) { cancelled.add(it) } + assertNull(store.load("u1")) + assertEquals(listOf("u1"), cancelled) + } + + @Test + fun `spares a recreate running under the same id`() { + // The acknowledged completion belongs to the id's PREVIOUS life. The + // manifest now holds a recreate's unaccepted parts, and a worker can be + // mid-transfer. A work cancel or a blob delete here would destroy its + // bytes. + val store = ChunkedManifestStore(tmp.newFolder()) + store.save(manifest("u1", accepted = false)) + val cancelled = mutableListOf() + releaseAckedCompletions(listOf("u1"), store) { cancelled.add(it) } + assertNotNull(store.load("u1")) + assertTrue(cancelled.isEmpty()) + } + + @Test + fun `ignores ids with no manifest (simple uploads)`() { + val store = ChunkedManifestStore(tmp.newFolder()) + val cancelled = mutableListOf() + releaseAckedCompletions(listOf("raw-upload"), store) { cancelled.add(it) } + assertTrue(cancelled.isEmpty()) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt new file mode 100644 index 00000000..8c909d62 --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedEngineTest.kt @@ -0,0 +1,181 @@ +package ai.openspace.backgroundupload + +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.yield +import org.junit.Assert.assertEquals +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Test +import java.util.concurrent.ConcurrentHashMap + +class ChunkedEngineTest { + + // runBlocking is single-threaded, so the overlap is deterministic. Every + // executor suspends at yield(). Thus all launchable siblings start before any + // executor finishes. + private class Tracker { + var inFlight = 0 + var maxInFlight = 0 + val executions = ConcurrentHashMap() + + suspend fun execute(index: Int) { + executions.merge(index, 1, Int::plus) + inFlight++ + maxInFlight = maxOf(maxInFlight, inFlight) + yield() + yield() + inFlight-- + } + } + + @Test + fun `runs every part exactly once`() = runBlocking { + val tracker = Tracker() + ChunkedEngine.run((0 until 10).toList()) { tracker.execute(it) } + assertEquals((0 until 10).associateWith { 1 }, tracker.executions.toMap()) + } + + @Test + fun `never more than WINDOW parts in flight`() = runBlocking { + val tracker = Tracker() + ChunkedEngine.run((0 until 10).toList()) { tracker.execute(it) } + assertEquals(ChunkedEngine.WINDOW, tracker.maxInFlight) + } + + @Test + fun `respects a smaller window`() = runBlocking { + val tracker = Tracker() + ChunkedEngine.run((0 until 5).toList(), window = 1) { tracker.execute(it) } + assertEquals(1, tracker.maxInFlight) + } + + @Test + fun `an empty part list completes immediately`() = runBlocking { + ChunkedEngine.run(emptyList()) { throw AssertionError("must not execute") } + } + + @Test + fun `a terminal part failure propagates and cancels the remaining parts`() { + val tracker = Tracker() + val thrown = assertThrows(IllegalStateException::class.java) { + runBlocking { + ChunkedEngine.run((0 until 10).toList()) { index -> + if (index == 0) throw IllegalStateException("part rejected") + tracker.execute(index) + } + } + } + assertEquals("part rejected", thrown.message) + assertTrue(tracker.executions.size < 10) + } + + @Test + fun `backoff grows exponentially and caps`() { + assertEquals(1_000, ChunkedEngine.backoffMs(1)) + assertEquals(2_000, ChunkedEngine.backoffMs(2)) + assertEquals(4_000, ChunkedEngine.backoffMs(3)) + assertEquals(60_000, ChunkedEngine.backoffMs(7)) + assertEquals(60_000, ChunkedEngine.backoffMs(100)) + // Defensive: a nonsense attempt number must not shift into a huge delay. + assertEquals(1_000, ChunkedEngine.backoffMs(0)) + } + + // MARK: - startAction + + private fun manifest(vararg accepted: Boolean) = ChunkedManifest( + id = "u1", + sourcePath = "/data/blob", + parts = accepted.mapIndexed { i, a -> + ChunkedManifest.Part( + url = "https://example.com/part?n=$i", + headers = emptyMap(), + start = i * 100L, + end = (i + 1) * 100L, + accepted = a, + ) + }, + accept = emptyList(), + expiresAt = 5_000, + wifiOnly = false, + noNotification = false, + createdAt = 1_000, + ) + + @Test + fun `no manifest at start is a silent success, never a journaled error`() { + // A completed ack or removeUpload deleted the manifest while this run sat + // in the queue. That is a legitimate end, already settled. + assertEquals(ChunkedEngine.StartAction.NO_MANIFEST, ChunkedEngine.startAction(null)) + } + + @Test + fun `an all-accepted manifest re-reports completion instead of running`() { + assertEquals( + ChunkedEngine.StartAction.ALREADY_COMPLETE, + ChunkedEngine.startAction(manifest(true, true)), + ) + } + + @Test + fun `pending parts run the engine`() { + assertEquals(ChunkedEngine.StartAction.RUN, ChunkedEngine.startAction(manifest(true, false))) + } + + // MARK: - completionReport + + private fun completedEntry(uploadId: String) = EventJournal.Entry( + eventId = "e-$uploadId", + uploadId = uploadId, + type = "completed", + timestamp = 1, + ) + + @Test + fun `an unacked completed entry is re-emitted, never minted twice`() { + assertEquals( + ChunkedEngine.CompletionReport.ReEmit(completedEntry("u1")), + ChunkedEngine.completionReport(listOf(completedEntry("u1")), "u1", freshCompletion = false), + ) + assertEquals( + ChunkedEngine.CompletionReport.ReEmit(completedEntry("u1")), + ChunkedEngine.completionReport(listOf(completedEntry("u1")), "u1", freshCompletion = true), + ) + } + + @Test + fun `a fresh completion with nothing journaled mints a new entry`() { + assertEquals( + ChunkedEngine.CompletionReport.Mint, + ChunkedEngine.completionReport(emptyList(), "u1", freshCompletion = true), + ) + } + + @Test + fun `a trailing run over an acked completion reports nothing`() { + // The trailing run raced ackEvents. The journal entry is already gone, but + // the manifest still exists for a moment. An acknowledged completion means + // that nobody is owed an event. A minted event would be a duplicate + // 'completed' for an upload that the consumer already settled. + assertEquals( + ChunkedEngine.CompletionReport.None, + ChunkedEngine.completionReport(emptyList(), "u1", freshCompletion = false), + ) + } + + @Test + fun `another upload's completed entry does not satisfy the lookup`() { + assertEquals( + ChunkedEngine.CompletionReport.None, + ChunkedEngine.completionReport(listOf(completedEntry("other")), "u1", freshCompletion = false), + ) + } + + @Test + fun `only 5xx responses are transient`() { + assertTrue(ChunkedEngine.isTransientHttp(500)) + assertTrue(ChunkedEngine.isTransientHttp(599)) + for (code in listOf(400, 401, 403, 404, 409, 429, 499, 600)) { + assertEquals("code $code", false, ChunkedEngine.isTransientHttp(code)) + } + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt new file mode 100644 index 00000000..798852ef --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedManifestTest.kt @@ -0,0 +1,384 @@ +package ai.openspace.backgroundupload + +import ai.openspace.backgroundupload.UploadOutcome.AcceptRule +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertNull +import org.junit.Assert.assertThrows +import org.junit.Assert.assertTrue +import org.junit.Rule +import org.junit.Test +import org.junit.rules.TemporaryFolder +import java.io.File +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +class ChunkedManifestTest { + @get:Rule + val tmp = TemporaryFolder() + + private fun manifest( + id: String = "u1", + parts: List = listOf( + part(0, 100), + part(100, 250), + ), + expiresAt: Long = 5_000, + ) = ChunkedManifest( + id = id, + sourcePath = "/data/blob", + parts = parts, + accept = listOf(AcceptRule(409, "already completed")), + expiresAt = expiresAt, + wifiOnly = false, + noNotification = false, + createdAt = 1_000, + ) + + private fun part(start: Long, end: Long, accepted: Boolean = false) = + ChunkedManifest.Part( + url = "https://example.com/part?start=$start", + headers = mapOf("Authorization" to "Bearer old"), + start = start, + end = end, + accepted = accepted, + ) + + // MARK: - Model + + @Test + fun `byte math is range-based`() { + val m = manifest(parts = listOf(part(0, 100, accepted = true), part(100, 250))) + assertEquals(250, m.totalBytes) + assertEquals(100, m.acceptedBytes) + } + + @Test + fun `completed only when every part is accepted`() { + val none = manifest() + assertFalse(none.allAccepted) + val partial = none.withPartAccepted(0) + assertFalse(partial.allAccepted) + val all = partial.withPartAccepted(1) + assertTrue(all.allAccepted) + assertEquals(emptyList(), all.pendingIndexes()) + assertEquals(listOf(1), partial.pendingIndexes()) + } + + @Test + fun `expiry is inclusive of the deadline`() { + val m = manifest(expiresAt = 5_000) + assertFalse(m.isExpired(4_999)) + assertTrue(m.isExpired(5_000)) + assertTrue(m.isExpired(5_001)) + } + + // MARK: - Reconcile: resume (same parts array) + + private val blobSize = 250L + + @Test + fun `resume replaces headers and deadline, keeps accepted parts and the moved source`() { + val stored = manifest().withPartAccepted(0) + val fresh = manifest(expiresAt = 99_000).copy( + sourcePath = "/ignored/by/reconcile", + createdAt = 42, + accept = listOf(AcceptRule(208)), + wifiOnly = true, + parts = stored.parts.map { it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) }, + ) + + val merged = stored.reconcile(fresh, running = false, blobSize = blobSize) + + assertEquals("Bearer new", merged.parts[0].headers["Authorization"]) + assertEquals(99_000, merged.expiresAt) + assertEquals(listOf(AcceptRule(208)), merged.accept) + assertTrue(merged.wifiOnly) + // Accepted statuses, ownership, and identity survive from the stored copy. + assertTrue(merged.parts[0].accepted) + assertFalse(merged.parts[1].accepted) + assertEquals("/data/blob", merged.sourcePath) + assertEquals(1_000, merged.createdAt) + } + + @Test + fun `resume is allowed while the upload is running`() { + // Fresh auth must reach a running worker's stalled parts. + val stored = manifest().withPartAccepted(0) + val fresh = manifest().copy( + parts = stored.parts.map { it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) }, + ) + val merged = stored.reconcile(fresh, running = true, blobSize = blobSize) + assertTrue(merged.parts[0].accepted) + assertEquals("Bearer new", merged.parts[1].headers["Authorization"]) + } + + @Test + fun `resume matches the same parts authored in a different order`() { + // Identical tiles, reordered, are the SAME upload: a resume, never a + // recreate (running = true would reject a recreate). Accepted flags follow + // the range, not the array index. + val stored = manifest().withPartAccepted(0) + val fresh = manifest().copy( + parts = listOf(stored.parts[1], stored.parts[0]).map { + it.copy(headers = mapOf("Authorization" to "Bearer new"), accepted = false) + }, + ) + val merged = stored.reconcile(fresh, running = true, blobSize = blobSize) + assertTrue(merged.parts.first { it.start == 0L }.accepted) + assertFalse(merged.parts.first { it.start == 100L }.accepted) + assertEquals("Bearer new", merged.parts[0].headers["Authorization"]) + } + + // MARK: - Reconcile: recreate (different parts array) + + @Test + fun `recreate from a stalled upload replaces parts and resets every status`() { + // The consumer re-authored under a fresh server uploadId: new urls, a new + // split, and fresh headers, accept, and expiresAt. The owned bytes stay. + val stored = manifest().withPartAccepted(0) + val fresh = manifest( + parts = listOf( + part(0, 120).copy(url = "https://example.com/v2?part=1"), + part(120, 250).copy(url = "https://example.com/v2?part=2"), + ), + expiresAt = 99_000, + ).copy(sourcePath = "/ignored/by/reconcile", createdAt = 42, accept = listOf(AcceptRule(208))) + + val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) + + assertTrue(recreated.parts.none { it.accepted }) + assertEquals(listOf("https://example.com/v2?part=1", "https://example.com/v2?part=2"), recreated.parts.map { it.url }) + assertEquals(99_000, recreated.expiresAt) + assertEquals(listOf(AcceptRule(208)), recreated.accept) + // Ownership survives. The blob is reused for the full re-upload. + assertEquals("/data/blob", recreated.sourcePath) + assertEquals(1_000, recreated.createdAt) + } + + @Test + fun `recreate with the same ranges but new urls also resets statuses`() { + // New part urls embed a new server uploadId, even when the split is + // identical. Nothing sent under the old id counts for the new one. + val stored = manifest().withPartAccepted(0) + val fresh = manifest( + parts = listOf( + part(0, 100).copy(url = "https://example.com/v2?part=1"), + part(100, 250).copy(url = "https://example.com/v2?part=2"), + ), + ) + val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) + assertTrue(recreated.parts.none { it.accepted }) + } + + @Test + fun `recreate is rejected while the upload is running`() { + val stored = manifest() + val fresh = manifest(parts = listOf(part(0, 250).copy(url = "https://example.com/v2"))) + assertThrows(ChunkedManifest.ReconcileException::class.java) { + stored.reconcile(fresh, running = true, blobSize = blobSize) + } + } + + @Test + fun `recreate rejects parts that do not tile the blob exactly`() { + val stored = manifest() + for ( + bad in listOf( + listOf(part(0, 100), part(150, 250)), // gap + listOf(part(0, 150), part(100, 250)), // overlap + listOf(part(50, 250)), // does not start at 0 + listOf(part(0, 200)), // short of the blob size + listOf(part(0, 100), part(100, 251)), // past the blob size + ) + ) { + assertThrows(ChunkedManifest.ReconcileException::class.java) { + stored.reconcile(manifest(parts = bad), running = false, blobSize = blobSize) + } + } + } + + @Test + fun `recreate accepts parts authored in any order`() { + val stored = manifest() + val fresh = manifest(parts = listOf(part(100, 250), part(0, 100)).map { it.copy(url = it.url + "&v=2") }) + val recreated = stored.reconcile(fresh, running = false, blobSize = blobSize) + assertEquals(2, recreated.parts.size) + } + + @Test + fun `tilesExactly covers the edge shapes`() { + assertTrue(ChunkedManifest.tilesExactly(listOf(part(0, 250)), 250)) + assertFalse(ChunkedManifest.tilesExactly(emptyList(), 0)) + assertFalse(ChunkedManifest.tilesExactly(listOf(part(0, 0)), 0)) // empty range + assertFalse(ChunkedManifest.tilesExactly(listOf(part(0, 250)), 300)) + } + + // MARK: - Create validation + + @Test + fun `create accepts parts that tile the blob exactly`() { + val m = manifest() + assertEquals(m, ChunkedManifest.validatedForCreate(m, blobSize)) + } + + @Test + fun `create rejects parts that do not tile the blob`() { + for ( + bad in listOf( + listOf(part(0, 100), part(150, 250)), // gap + listOf(part(0, 150), part(100, 250)), // overlap + listOf(part(50, 250)), // does not start at 0 + listOf(part(0, 200)), // short of the blob size + listOf(part(0, 100), part(100, 251)), // past the blob size + ) + ) { + assertThrows(ChunkedManifest.ReconcileException::class.java) { + ChunkedManifest.validatedForCreate(manifest(parts = bad), blobSize) + } + } + } + + @Test + fun `a rejected create writes no manifest, leaving the blob adoptable`() { + // startUpload validates AFTER takeOwnership moved the bytes. The throw + // propagates out of compute before a save. Thus the blob sits ownerless at + // its path. That is exactly what takeOwnership's orphan branch adopts on + // the corrected retry. + val store = ChunkedManifestStore(tmp.newFolder()) + store.blobFile("u1").apply { parentFile!!.mkdirs() }.writeText("owned bytes") + assertThrows(ChunkedManifest.ReconcileException::class.java) { + store.compute("u1") { ChunkedManifest.validatedForCreate(manifest(), 999L) } + } + assertNull(store.load("u1")) + assertTrue(store.blobFile("u1").exists()) + } + + // MARK: - Store + + @Test + fun `save then load round-trips, across store instances`() { + val dir = tmp.newFolder() + val m = manifest().withPartAccepted(1) + ChunkedManifestStore(dir).save(m) + // A new instance over the same dir is what a process relaunch looks like. + assertEquals(m, ChunkedManifestStore(dir).load("u1")) + } + + @Test + fun `load returns null for an unknown id`() { + assertNull(ChunkedManifestStore(tmp.newFolder()).load("nope")) + } + + @Test + fun `update persists the transformed manifest`() { + val store = ChunkedManifestStore(tmp.newFolder()) + store.save(manifest()) + val updated = store.update("u1") { it.withPartAccepted(0) } + assertTrue(updated!!.parts[0].accepted) + assertTrue(store.load("u1")!!.parts[0].accepted) + } + + @Test + fun `update of a missing manifest returns null`() { + assertNull(ChunkedManifestStore(tmp.newFolder()).update("nope") { it }) + } + + @Test + fun `compute creates when no manifest exists`() { + val store = ChunkedManifestStore(tmp.newFolder()) + val created = store.compute("u1") { existing -> + assertNull(existing) + manifest() + } + assertEquals(created, store.load("u1")) + } + + @Test + fun `compute holds the store lock across load, transform, and save`() { + // The startUpload reconcile and a running worker's markAccepted race. If + // the lock did not span all three steps, the update below could land + // between compute's load and save, and it would be erased from disk. When + // they are serialized, both effects must survive. + val store = ChunkedManifestStore(tmp.newFolder()) + store.save(manifest()) + val inTransform = CountDownLatch(1) + val computing = Thread { + store.compute("u1") { existing -> + inTransform.countDown() + Thread.sleep(300) // hold the lock with load done and save not yet run + existing!!.copy(expiresAt = 99_000) + } + }.apply { start() } + assertTrue(inTransform.await(5, TimeUnit.SECONDS)) + val updating = Thread { store.update("u1") { it.withPartAccepted(0) } }.apply { start() } + computing.join() + updating.join() + val final = store.load("u1")!! + assertEquals(99_000, final.expiresAt) + assertTrue(final.parts[0].accepted) + } + + @Test + fun `a throwing compute transform propagates and writes nothing`() { + val store = ChunkedManifestStore(tmp.newFolder()) + store.save(manifest()) + assertThrows(ChunkedManifest.ReconcileException::class.java) { + store.compute("u1") { throw ChunkedManifest.ReconcileException("rejected") } + } + assertEquals(manifest(), store.load("u1")) + } + + @Test + fun `contains tracks save and remove`() { + val store = ChunkedManifestStore(tmp.newFolder()) + assertFalse(store.contains("u1")) + store.save(manifest()) + assertTrue(store.contains("u1")) + store.remove("u1") + assertFalse(store.contains("u1")) + } + + @Test + fun `remove deletes the manifest and the blob`() { + val store = ChunkedManifestStore(tmp.newFolder()) + store.save(manifest()) + store.blobFile("u1").writeText("bytes") + store.remove("u1") + assertNull(store.load("u1")) + assertFalse(store.blobFile("u1").exists()) + } + + @Test + fun `remove of an unknown id is a no-op`() { + ChunkedManifestStore(tmp.newFolder()).remove("simple-upload-id") + } + + @Test + fun `ids with filesystem-hostile characters round-trip`() { + val store = ChunkedManifestStore(tmp.newFolder()) + val id = "a/b:c dü..\\e" + store.save(manifest(id = id)) + assertEquals(id, store.load(id)!!.id) + store.remove(id) + assertNull(store.load(id)) + } + + @Test + fun `a corrupt manifest reads as absent, not fatal`() { + val dir = tmp.newFolder() + val store = ChunkedManifestStore(dir) + store.save(manifest()) + File(File(dir, dir.list()!!.first()), "manifest.json").writeText("{not json") + assertNull(store.load("u1")) + assertEquals(emptyList(), store.all()) + } + + @Test + fun `all lists every stored manifest`() { + val store = ChunkedManifestStore(tmp.newFolder()) + store.save(manifest(id = "u1")) + store.save(manifest(id = "u2").withPartAccepted(0)) + assertEquals(setOf("u1", "u2"), store.all().map { it.id }.toSet()) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt b/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt new file mode 100644 index 00000000..0b65d54b --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/ChunkedWorkerGateTest.kt @@ -0,0 +1,57 @@ +package ai.openspace.backgroundupload + +import org.junit.After +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class ChunkedWorkerGateTest { + private val a = Any() + private val b = Any() + + @After + fun tearDown() { + // The gate is a process-wide singleton. Leave nothing for other tests. + ChunkedWorkerGate.release("u1", a) + ChunkedWorkerGate.release("u1", b) + ChunkedWorkerGate.release("u2", a) + } + + @Test + fun `a second worker for the same id must wait`() { + assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) + // The replacement worker after a cancel-then-start: it must not run a part + // PUT while the cancelled worker still holds the id. + assertFalse(ChunkedWorkerGate.tryAcquire("u1", b)) + ChunkedWorkerGate.release("u1", a) + assertTrue(ChunkedWorkerGate.tryAcquire("u1", b)) + } + + @Test + fun `reacquiring with the same token is idempotent`() { + assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) + assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) + } + + @Test + fun `a stale release cannot evict a successor`() { + assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) + ChunkedWorkerGate.release("u1", a) + assertTrue(ChunkedWorkerGate.tryAcquire("u1", b)) + ChunkedWorkerGate.release("u1", a) // the old worker's finally, arriving late + assertTrue(ChunkedWorkerGate.isRunning("u1")) + assertFalse(ChunkedWorkerGate.tryAcquire("u1", a)) + } + + @Test + fun `ids are independent and isRunning tracks the holder`() { + assertFalse(ChunkedWorkerGate.isRunning("u1")) + assertTrue(ChunkedWorkerGate.tryAcquire("u1", a)) + assertTrue(ChunkedWorkerGate.isRunning("u1")) + assertFalse(ChunkedWorkerGate.isRunning("u2")) + assertTrue(ChunkedWorkerGate.tryAcquire("u2", a)) + ChunkedWorkerGate.release("u1", a) + assertFalse(ChunkedWorkerGate.isRunning("u1")) + assertTrue(ChunkedWorkerGate.isRunning("u2")) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt index b9375ce0..187349ed 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/UploadOutcomeTest.kt @@ -1,5 +1,6 @@ package ai.openspace.backgroundupload +import ai.openspace.backgroundupload.UploadOutcome.AcceptRule import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue @@ -10,28 +11,51 @@ class UploadOutcomeTest { @Test fun `2xx is accepted`() { - assertTrue(UploadOutcome.isAccepted(200, listOf())) - assertTrue(UploadOutcome.isAccepted(204, listOf())) - assertTrue(UploadOutcome.isAccepted(299, listOf())) + assertTrue(UploadOutcome.isAccepted(200, "", listOf())) + assertTrue(UploadOutcome.isAccepted(204, "", listOf())) + assertTrue(UploadOutcome.isAccepted(299, "", listOf())) } @Test fun `non-2xx is not accepted by default`() { - assertFalse(UploadOutcome.isAccepted(199, listOf())) - assertFalse(UploadOutcome.isAccepted(300, listOf())) - assertFalse(UploadOutcome.isAccepted(404, listOf())) - assertFalse(UploadOutcome.isAccepted(500, listOf())) + assertFalse(UploadOutcome.isAccepted(199, "", listOf())) + assertFalse(UploadOutcome.isAccepted(300, "", listOf())) + assertFalse(UploadOutcome.isAccepted(404, "", listOf())) + assertFalse(UploadOutcome.isAccepted(500, "", listOf())) } @Test - fun `non-2xx listed in acceptStatus is accepted`() { - assertTrue(UploadOutcome.isAccepted(409, listOf(409))) - assertTrue(UploadOutcome.isAccepted(404, listOf(404, 409))) + fun `a status-only rule accepts its status regardless of body`() { + val rules = listOf(AcceptRule(409)) + assertTrue(UploadOutcome.isAccepted(409, "anything", rules)) + assertTrue(UploadOutcome.isAccepted(409, null, rules)) } @Test - fun `acceptStatus does not accept unlisted codes`() { - assertFalse(UploadOutcome.isAccepted(500, listOf(409))) + fun `rules do not accept unlisted codes`() { + assertFalse(UploadOutcome.isAccepted(500, "", listOf(AcceptRule(409)))) + } + + // The backend's 409 carries several meanings, and only the message shows the + // difference. bodyIncludes is what separates the success meanings from the + // bug meanings. + @Test + fun `bodyIncludes narrows a rule to matching bodies`() { + val rules = listOf(AcceptRule(409, bodyIncludes = "already completed")) + assertTrue(UploadOutcome.isAccepted(409, "part already completed", rules)) + assertFalse(UploadOutcome.isAccepted(409, "part number mismatch", rules)) + assertFalse(UploadOutcome.isAccepted(409, null, rules)) + } + + @Test + fun `any matching rule accepts`() { + val rules = listOf( + AcceptRule(409, bodyIncludes = "already completed"), + AcceptRule(208), + ) + assertTrue(UploadOutcome.isAccepted(208, "", rules)) + assertTrue(UploadOutcome.isAccepted(409, "already completed", rules)) + assertFalse(UploadOutcome.isAccepted(410, "already completed", rules)) } @Test diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt new file mode 100644 index 00000000..bef1858c --- /dev/null +++ b/android/src/test/java/ai/openspace/backgroundupload/UploadStatesTest.kt @@ -0,0 +1,84 @@ +package ai.openspace.backgroundupload + +import androidx.work.WorkInfo.State.BLOCKED +import androidx.work.WorkInfo.State.CANCELLED +import androidx.work.WorkInfo.State.ENQUEUED +import androidx.work.WorkInfo.State.FAILED +import androidx.work.WorkInfo.State.RUNNING +import androidx.work.WorkInfo.State.SUCCEEDED +import androidx.work.ListenableWorker +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +// getAllUploads must return ONE row per upload id, although WorkManager can +// hold several rows for it (finished chains linger for roughly a day, and +// APPEND_OR_REPLACE resumes add rows). The state vocabulary is the same as +// iOS's. +class UploadStatesTest { + + @Test + fun `a live row wins for a chunked upload`() { + assertEquals("running", chunkedUploadState(listOf(CANCELLED, RUNNING), allAccepted = false)) + assertEquals("running", chunkedUploadState(listOf(RUNNING, BLOCKED), allAccepted = false)) + assertEquals("pending", chunkedUploadState(listOf(FAILED, ENQUEUED), allAccepted = false)) + assertEquals("pending", chunkedUploadState(listOf(BLOCKED), allAccepted = false)) + } + + @Test + fun `with no live row the manifest speaks, never a lingering finished row`() { + // A cancelled chunked upload keeps its manifest. Its truthful state is + // stalled-awaiting-resume ("error"), not "cancelled". iOS's getAllUploads + // never reports "cancelled" for a lingering upload. + assertEquals("error", chunkedUploadState(listOf(CANCELLED), allAccepted = false)) + assertEquals("error", chunkedUploadState(listOf(FAILED), allAccepted = false)) + assertEquals("error", chunkedUploadState(emptyList(), allAccepted = false)) + assertEquals("completed", chunkedUploadState(listOf(SUCCEEDED), allAccepted = true)) + assertEquals("completed", chunkedUploadState(emptyList(), allAccepted = true)) + // The row that a run leaves after it journals a terminal error is + // SUCCEEDED (see terminalErrorResult). The manifest, not the row, carries + // the outcome. + assertEquals("error", chunkedUploadState(listOf(SUCCEEDED), allAccepted = false)) + } + + @Test + fun `a journaled terminal error still succeeds the row`() { + // WorkManager marks the dependents of a FAILED prerequisite FAILED without + // a run. Thus a resume appended during a failing run's teardown would + // silently never run. The journal and the manifest are the outcome record, + // never the row state. + assertTrue(terminalErrorResult() is ListenableWorker.Result.Success) + } + + @Test + fun `cancel reports from the module only when no worker is running`() { + assertTrue(cancelReportsFromModule(listOf(ENQUEUED))) + // An appended chain's dependent is BLOCKED, not ENQUEUED. It is still + // never-started, and it is still owed a module-side 'cancelled'. + assertTrue(cancelReportsFromModule(listOf(BLOCKED))) + assertTrue(cancelReportsFromModule(listOf(ENQUEUED, BLOCKED))) + // A RUNNING worker's stop handler owns the report. + assertFalse(cancelReportsFromModule(listOf(RUNNING))) + assertFalse(cancelReportsFromModule(listOf(RUNNING, BLOCKED))) + assertFalse(cancelReportsFromModule(emptyList())) + } + + @Test + fun `a queued successor suppresses another append`() { + assertTrue(hasQueuedSuccessor(listOf(RUNNING, BLOCKED))) + assertTrue(hasQueuedSuccessor(listOf(ENQUEUED))) + assertFalse(hasQueuedSuccessor(listOf(RUNNING))) + assertFalse(hasQueuedSuccessor(listOf(SUCCEEDED, FAILED, CANCELLED))) + assertFalse(hasQueuedSuccessor(emptyList())) + } + + @Test + fun `a simple upload reports its live row first, then the most conclusive finished one`() { + assertEquals("running", simpleUploadState(listOf(CANCELLED, RUNNING))) + assertEquals("pending", simpleUploadState(listOf(SUCCEEDED, ENQUEUED))) + assertEquals("completed", simpleUploadState(listOf(CANCELLED, SUCCEEDED))) + assertEquals("error", simpleUploadState(listOf(CANCELLED, FAILED))) + assertEquals("cancelled", simpleUploadState(listOf(CANCELLED))) + } +} diff --git a/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt b/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt index 85eb5a06..a99178d1 100644 --- a/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt +++ b/android/src/test/java/ai/openspace/backgroundupload/UploadTest.kt @@ -1,6 +1,7 @@ package ai.openspace.backgroundupload import com.google.gson.Gson +import org.junit.Assert.assertEquals import org.junit.Assert.assertFalse import org.junit.Assert.assertTrue import org.junit.Test @@ -14,7 +15,7 @@ class UploadTest { path = "/tmp/file", method = "POST", wifiOnly = false, - acceptStatus = listOf(), + accept = listOf(), headers = mapOf(), noNotification = noNotification, ) @@ -40,4 +41,72 @@ class UploadTest { json.remove(Upload::noNotification.name) assertTrue(gson.fromJson(json, Upload::class.java).showsNotification) } + + // One build can enqueue a WorkManager job, and the next build can replay it. + // This is the exact JSON shape that a v8 build serialized into input data + // (Gson.toJson of the v8 Upload model): `acceptStatus: List`, and no + // `accept`. Gson does not use the constructor. Thus, without normalized(), + // the replayed object's `accept` is NULL, and the worker NPEs after the file + // has fully transmitted. WorkManager then re-runs it and re-sends the whole + // file. + private val v8JobJson = """ + { + "id": "u1", + "url": "https://example.com/upload", + "path": "/tmp/file", + "method": "PUT", + "maxRetries": 5, + "wifiOnly": false, + "acceptStatus": [409, 208], + "headers": {"Authorization": "Bearer t"}, + "notificationId": 123456, + "notificationTitle": "Uploading…", + "notificationTitleNoInternet": "Waiting for connection…", + "notificationTitleNoWifi": "Waiting for Wi-Fi…", + "notificationChannel": "background-upload", + "noNotification": false + } + """ + + @Test + fun `a replayed v8 job maps acceptStatus to accept rules and is safe to run`() { + val replayed = gson.fromJson(v8JobJson, Upload::class.java).normalized() + assertEquals( + listOf(UploadOutcome.AcceptRule(409), UploadOutcome.AcceptRule(208)), + replayed.accept, + ) + // The worker-facing calls that NPE'd on the un-normalized object. + assertTrue(UploadOutcome.isAccepted(409, "duplicate", replayed.accept)) + assertFalse(UploadOutcome.isAccepted(400, "", replayed.accept)) + assertEquals("u1", replayed.id) + assertEquals(mapOf("Authorization" to "Bearer t"), replayed.headers) + assertTrue(replayed.showsNotification) + } + + @Test + fun `a replayed v8 job with an empty acceptStatus gets no rules`() { + val json = gson.fromJson(v8JobJson, com.google.gson.JsonObject::class.java) + json.add("acceptStatus", com.google.gson.JsonArray()) + val replayed = gson.fromJson(json, Upload::class.java).normalized() + assertEquals(emptyList(), replayed.accept) + assertTrue(UploadOutcome.isAccepted(200, "", replayed.accept)) + } + + @Test + fun `a job with neither accept nor acceptStatus normalizes to no rules`() { + val json = gson.fromJson(v8JobJson, com.google.gson.JsonObject::class.java) + json.remove("acceptStatus") + val replayed = gson.fromJson(json, Upload::class.java).normalized() + assertEquals(emptyList(), replayed.accept) + assertFalse(UploadOutcome.isAccepted(409, "duplicate", replayed.accept)) + } + + @Test + fun `normalized passes a current-shape job through unchanged`() { + val current = upload(noNotification = true).copy( + accept = listOf(UploadOutcome.AcceptRule(409, "already completed")), + ) + val replayed = gson.fromJson(gson.toJson(current), Upload::class.java).normalized() + assertEquals(current, replayed) + } } diff --git a/src/types.ts b/src/types.ts index 620b317e..77c186ea 100644 --- a/src/types.ts +++ b/src/types.ts @@ -57,6 +57,11 @@ export interface ErrorData extends TerminalEventData { * is missing or unreadable on disk, so retrying can never succeed. */ errorKind?: ErrorKind; + /** + * Chunked uploads: the index into `parts` of the failing part, when one + * part's response caused the error. + */ + partIndex?: number; } export interface CancelledData extends TerminalEventData { @@ -76,8 +81,10 @@ export type JournaledEvent = CompletedData | ErrorData | CancelledData; export interface UploadSnapshot { id: UploadId; state: 'pending' | 'running' | 'completed' | 'error' | 'cancelled'; - bytesSent?: number; // iOS only - totalBytes?: number; // iOS only + /** iOS: bytes sent so far. Android: a chunked upload's accepted bytes. */ + bytesSent?: number; + /** The total payload bytes. On iOS always; on Android for chunked uploads. */ + totalBytes?: number; } export type UploadOptions = { From 891dfc68055261c4e99bf4f46d0a6623bd2d83e2 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 3 Sep 2026 10:52:37 -0400 Subject: [PATCH 3/4] iOS chunked-upload engine (#34) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the iOS engine for chunked uploads. The JS-visible behavior is the same as Android. The transport is a background `URLSession`. A background session can send only files. Thus the engine writes one temporary file for each part in flight, and deletes it when the part completes. The transient disk usage stays near 60 MB. There is no second full copy of the source file. The engine keeps a sliding window of 3 part tasks enqueued in the session. The session daemon continues the tasks when the app is suspended or terminated. When iOS relaunches the app, the engine reconciles the manifest with the daemon's task list, and then fills the window again. The background completion-handler flow is kept. Review fixes in this slice: - Same-id `startUpload` races are serialized natively. - An incarnation token stops the late callbacks of a removed upload. Without the token, those callbacks could corrupt a recreated upload with the same id. - The recreate rule and the stale temporary-file edges are covered. 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-authored-by: Claude Fable 5 --- ios/ChunkedCoordinator.swift | 721 +++++++++++++++++++++++++++++++++++ ios/ChunkedEngine.swift | 81 ++++ ios/ChunkedManifest.swift | 404 ++++++++++++++++++++ ios/EventJournal.swift | 9 +- ios/RNBackgroundUpload.swift | 299 +++++++++++++-- ios/RNFileUploader.mm | 18 +- ios/TaskMap.swift | 59 ++- ios/UploadOutcome.swift | 40 ++ 8 files changed, 1584 insertions(+), 47 deletions(-) create mode 100644 ios/ChunkedCoordinator.swift create mode 100644 ios/ChunkedEngine.swift create mode 100644 ios/ChunkedManifest.swift create mode 100644 ios/UploadOutcome.swift diff --git a/ios/ChunkedCoordinator.swift b/ios/ChunkedCoordinator.swift new file mode 100644 index 00000000..20121ed5 --- /dev/null +++ b/ios/ChunkedCoordinator.swift @@ -0,0 +1,721 @@ +import Foundation + +/// Runs chunked uploads against the background sessions. It keeps the sliding +/// window of part tasks enqueued with the daemon. It evaluates the outcome of +/// each part. After a relaunch, it reconciles the durable [ChunkedManifest] +/// with the tasks that the daemon still holds. +/// +/// Every state transition occurs on one serial queue. The queue enforces the +/// invariants that the design marks binding: at most [ChunkedEngine.window] +/// part tasks are enqueued per upload, and never two for the same part index. +/// `inFlight` maps each enqueued part to the task key that owns it. Only +/// refill, on this queue, creates tasks. +final class ChunkedCoordinator { + + // The singleton that owns the background sessions. It outlives this object. + // Both live for the whole process. + private unowned let uploader: RNBackgroundUpload + + private let queue = DispatchQueue(label: "ai.openspace.rnbgupload.chunked") + + // partIndex -> the TaskMap key of the one task that may be in flight for it. + // The key lets us tell a superseded task's late completion (possible around + // a relaunch reconcile) apart from the live task's completion. + private var inFlight: [String: [Int: String]] = [:] + // The uploads whose in-flight set we rebuild from the daemon now. Refill is + // blocked until the rebuild lands. Thus a stale snapshot can never + // double-enqueue. The token makes overlapping reconciles safe: only the + // latest reconcile may apply its snapshot. An earlier snapshot could miss + // tasks enqueued after it was taken. To apply it would re-enqueue their + // part indexes. + private var reconcileToken: [String: UUID] = [:] + private var cooldownUntil: [String: [Int: Double]] = [:] // epoch ms + private var transientAttempts: [String: [Int: Int]] = [:] + private var expiryArmed: Set = [] + // The in-flight bytes per part index. They feed the byte-weighted + // aggregate progress. + private var partSent: [String: [Int: Int64]] = [:] + // A cache of the stored manifests, refreshed on every load. The progress + // path reads it. Thus didSendBodyData never touches the disk. + private var manifests: [String: ChunkedManifest] = [:] + + private static let progressThrottle: TimeInterval = 0.5 // seconds, per upload + private let progressLock = NSLock() + private var lastProgressAt: [String: TimeInterval] = [:] + + init(uploader: RNBackgroundUpload) { + self.uploader = uploader + } + + private func nowMs() -> Double { Date().timeIntervalSince1970 * 1000 } + + // MARK: - Entry points (module methods) + + /// Starts, or resumes, a chunked upload. The durable manifest makes the call + /// idempotent. A first call takes ownership of the source file (an O(1) + /// rename into the library's directory) and saves the manifest BEFORE any + /// task is enqueued. A new call with the same id reconciles instead. The + /// same parts resume: the stored headers are replaced, and accepted parts + /// are skipped. Different parts recreate the upload, per the design's rule + /// (see ChunkedManifest.reconciled). Crash recovery, resume after a stop, + /// and resume with fresh auth are all this same call. + /// + /// Every rejection-type validation runs BEFORE the source is consumed. The + /// parse throws first, and a reconcile never touches the source (`path` is + /// ignored once a manifest exists). One rejection is possible after the + /// move: the manifest save can fail. That leaves the blob adoptable. A + /// retry with the same id finds the blob at the blob path and proceeds (see + /// takeOwnership). + func startUpload(_ options: [String: Any], + resolve: @escaping (String) -> Void, + reject: @escaping (String) -> Void) { + queue.async { + do { + let incoming = try ChunkedManifest.parse(options, createdAt: self.nowMs()) + let id = incoming.id + let manifest: ChunkedManifest + if let existing = ChunkedStore.load(id) { + // "Running" per the design's recreate rule: not stalled (no + // journaled terminal error or cancel that awaits this resume) and + // not past its deadline. Everything else rejects a different parts + // array. That includes part tasks live with the daemon, and + // finished-but-unacked. + let running = !existing.stalled && !existing.isExpired(self.nowMs()) + manifest = try existing.reconciled( + with: incoming, running: running, blobSize: ChunkedStore.blobSize(id)) + if manifest.incarnation != existing.incarnation { + // This is a recreate. The in-flight byte counts belong to the + // replaced parts. reconcileLocked below cancels the old + // incarnation's tasks, and does not adopt them. enqueuePart + // sweeps its temp files. + self.partSent[id] = nil + } + } else { + guard let path = options["path"] as? String else { + throw ChunkedManifest.ParseError(message: "Missing 'path'") + } + try self.takeOwnership(path: path, id: id) + // The same rule as recreate, and as Android's validatedForCreate: + // the parts must tile [0, blob size) exactly. A partial or + // overlapping cover would silently upload wrong bytes. This throws + // BEFORE the manifest is saved and before anything is enqueued. + // Thus the moved blob stays adoptable by a corrected retry with the + // same id (see takeOwnership). + let blobSize = ChunkedStore.blobSize(id) + guard ChunkedManifest.tilesExactly(incoming.parts, size: blobSize) else { + throw ChunkedManifest.ParseError( + message: "chunked upload '\(id)' parts must tile exactly [0, \(blobSize))") + } + manifest = incoming + } + try ChunkedStore.save(manifest) + self.manifests[id] = manifest + // A fresh call gets a fresh retry budget. The persisted per-part + // rejection counts reset in the parts rebuild above (reconciled or + // parse). + self.transientAttempts[id] = nil + self.cooldownUntil[id] = nil + self.reconcileLocked(id, resumedByStart: true) + resolve(id) + } catch { + reject(error.localizedDescription) + } + } + } + + /// Rebuilds every stored upload's in-flight set from the daemon, and then + /// refills. Called when the sessions are created or recreated: an app + /// relaunch, a JS reload, or the background-wake path through + /// `RNBackgroundUpload.shared`. + func reconcileAll() { + // Claim the system's background completion handlers BEFORE the reconcile + // is queued. After a relaunch, the replayed didCompleteWithError callbacks + // run unowned and never refill. Thus this chain is the only refill. + // Nothing else stops urlSessionDidFinishEvents from handing the system + // its handler, and the app its suspension, before one new part task is + // enqueued. The risk is largest exactly when every enqueued part finished + // while the app was dead: zero daemon tasks left, no future wake, and a + // silent stall. The claim provably precedes any drain: a relaunch reaches + // this point inside the init of `shared`, and the AppDelegate hook + // finishes that init before it stores the handler. + RNBackgroundUpload.deferBackgroundCompletionHandlers() + queue.async { + let group = DispatchGroup() + for manifest in ChunkedStore.all() { + self.manifests[manifest.id] = manifest + group.enter() + self.reconcileLocked(manifest.id, resumedByStart: false) { group.leave() } + } + group.notify(queue: self.queue) { + // Every upload's post-reconcile refill has resumed its tasks. The + // handlers can drain now. + RNBackgroundUpload.releaseBackgroundCompletionHandlers() + } + } + } + + /// Cancels a chunked upload. It journals one 'cancelled' (user) terminal + /// and stalls the upload. The manifest and the bytes are kept. Thus the + /// next startUpload resumes. Completion receives nil when the id has no + /// manifest (not a chunked upload). It receives false when nothing runs + /// (the upload is already terminal). + func cancel(_ id: String, completion: @escaping (Bool?) -> Void) { + queue.async { + guard let manifest = self.latest(id) else { completion(nil); return } + if manifest.stalled || manifest.allAccepted { completion(false); return } + var entry = JournaledEvent( + eventId: UUID().uuidString, id: id, type: "cancelled", timestamp: self.nowMs()) + entry.cancelReason = "user" + self.stall(id, entry: entry) + completion(true) + } + } + + /// An explicit release. It cancels the in-flight part tasks, with no + /// terminal event: the consumer lets go, and awaits no outcome. It deletes + /// the manifest, the moved bytes, and all part temp files. + func remove(_ id: String, completion: @escaping () -> Void) { + queue.async { + if self.latest(id) != nil { self.cancelTasks(for: id) } + ChunkedStore.remove(id) + self.clearState(id) + completion() + } + } + + /// The one moment when the library may delete a chunked upload's bytes: the + /// consumer acknowledged its 'completed' terminal event. + func releaseCompleted(_ ids: [String], completion: @escaping () -> Void) { + queue.async { + for id in ids { + ChunkedStore.remove(id) + self.clearState(id) + } + completion() + } + } + + /// The chunked rows for getAllUploads: one aggregate row per manifest. The + /// part tasks are transport detail. bytesSent counts accepted parts only. + /// That is the durable number. + func snapshots(completion: @escaping ([[String: Any]]) -> Void) { + queue.async { + let rows = ChunkedStore.all().map { manifest -> [String: Any] in + let state: String + if manifest.allAccepted { + state = "completed" + } else if manifest.stalled { + state = "error" + } else if !(self.inFlight[manifest.id] ?? [:]).isEmpty { + state = "running" + } else { + state = "pending" + } + return ["id": manifest.id, + "state": state, + "bytesSent": manifest.acceptedBytes, + "totalBytes": manifest.totalBytes] + } + completion(rows) + } + } + + // MARK: - Delegate hooks (called by RNBackgroundUpload) + + func partProgress(id: String, part: Int, incarnation: String?, sent: Int64) { + let now = Date().timeIntervalSince1970 + progressLock.lock() + if let last = lastProgressAt[id], now - last < Self.progressThrottle { + progressLock.unlock() + return + } + lastProgressAt[id] = now + progressLock.unlock() + queue.async { + // A removed or replaced incarnation's task must not feed the aggregate. + guard let manifest = self.manifests[id], manifest.incarnation == incarnation else { return } + self.partSent[id, default: [:]][part] = sent + self.emitAggregateProgress(id, manifest) + } + } + + /// One part task finished (a foreground or background-wake delegate + /// callback). We evaluate the accept rules, update the manifest, delete the + /// temp file, and refill the window. It is synchronous on purpose: the + /// journal write for a terminal outcome must land before the delegate + /// callback returns. The simple-upload path obeys the same rule. + func handlePartCompletion(id: String, part: Int, incarnation: String?, taskKey: String, + statusCode: Int?, headers: [String: String], + body: String?, error: NSError?) { + queue.sync { + TaskMap.removeKey(taskKey) + let owned = inFlight[id]?[part] == taskKey + if owned { + inFlight[id]?[part] = nil + partSent[id]?[part] = nil + } + guard var manifest = latest(id) else { + // The upload was removed (removeUpload, or a completed ack) while + // this task was in flight. There is nothing left to report. + if owned { ChunkedStore.removePartFile(id, part) } + return + } + // A late callback from a removed-then-recreated or replaced + // incarnation. Its response is about byte ranges and URLs that this + // manifest no longer describes. Thus nothing about it, the accept flag + // included, may be written into the current manifest. Its temp file has + // the old token in its name. The sweep removes it when the current plan + // next materializes this index. + guard incarnation == manifest.incarnation else { + if owned { refill(id) } + return + } + // A part index that the manifest does not know (corrupt task metadata) + // must not crash the delegate. Drop the task's outcome and let refill + // plan again. + guard manifest.parts.indices.contains(part) else { + if owned { refill(id) } + return + } + + // Accept evaluation comes first. The server holds these bytes now, + // regardless of a concurrent stall or a superseded task in the same + // incarnation. If we lose the flag, we re-send a part that the server + // already has. + if error == nil, let statusCode, + UploadOutcome.isAccepted(statusCode, body: body, accept: manifest.accept) { + manifest = updateManifest(id) { $0.withPartAccepted(part) } + ?? manifest.withPartAccepted(part) + transientAttempts[id]?[part] = nil + cooldownUntil[id]?[part] = nil + if owned { ChunkedStore.removePartFile(id, part) } + // A stalled upload keeps the flag but reports nothing more. The + // journaled terminal stands until the next startUpload resume. That + // resume finds all parts accepted and completes without a re-send. + guard !manifest.stalled else { return } + if manifest.allAccepted { + finalizeCompleted(id, manifest, reemit: false) + } else { + emitAggregateProgress(id, manifest) + if owned { refill(id) } + } + return + } + + // A superseded task's failure carries no policy weight. The live task + // for this part drives the retries. But an UNOWNED task with no live + // replacement is a relaunch replay that runs before reconcile rebuilds + // ownership. If we drop its deterministic HTTP rejection, the part gets + // a fresh retry budget on every system wake. So count it, and let it + // trip the budget. The in-flight reconcile does the re-enqueueing. + guard owned else { + if inFlight[id]?[part] == nil, !manifest.stalled, !manifest.parts[part].accepted, + error == nil, let code = statusCode, !ChunkedEngine.isTransientHttp(code) { + recordRejection(id, part: part, manifest: manifest, code: code, + headers: headers, body: body, scheduleRetryInBudget: false) + } + return + } + ChunkedStore.removePartFile(id, part) // the retry builds the file again + // This is a duplicate of a part that a superseded task already + // delivered. The part is settled, whatever this task's outcome was. Its + // failure must not burn retries. + if manifest.parts[part].accepted { + if !manifest.stalled { refill(id) } + return + } + // A terminal is already journaled (a cancel, or a sibling part's + // stall). Swallow the fallout. + guard !manifest.stalled else { return } + + if let error, error.domain == NSURLErrorDomain, error.code == NSURLErrorCancelled { + // A user cancel journals and stalls in cancel() before the tasks are + // torn down. Thus a cancel here, with no stall, comes from the + // system. Retry it like a transient failure. + scheduleTransientRetry(id, part: part) + return + } + + if manifest.isExpired(nowMs()) { + stall(id, entry: expiredEntry(id)) + return + } + + if let error { + if RNBackgroundUpload.errorKind(for: error) == "file", + !FileManager.default.fileExists(atPath: ChunkedStore.blobURL(id).path) { + stall(id, entry: errorEntry( + id: id, error: "chunked source blob missing", errorKind: "file", partIndex: part)) + } else { + // This includes a lost temp part file. The retry rebuilds it from + // the blob. + scheduleTransientRetry(id, part: part) + } + return + } + + let code = statusCode ?? 0 + if ChunkedEngine.isTransientHttp(code) { + scheduleTransientRetry(id, part: part) + return + } + recordRejection(id, part: part, manifest: manifest, code: code, + headers: headers, body: body, scheduleRetryInBudget: true) + } + } + + /// The identity of a chunked part task, or nil for a simple upload's task. + /// taskDescription is primary. The persisted TaskMap entry, written before + /// the task first resumed, is the durable fallback. `incarnation` is the + /// manifest token that the task was created under. It is nil only for + /// corrupt metadata, and the consumers treat nil as a mismatch. + static func partRef(_ session: URLSession, _ task: URLSessionTask) + -> (id: String, part: Int, incarnation: String?)? { + if let ref = ChunkedEngine.parseTaskDescription(task.taskDescription) { return ref } + if let meta = TaskMap.meta(forKey: TaskMap.key(session, task)), let part = meta.partIndex { + return (meta.id, part, meta.incarnation) + } + return nil + } + + // MARK: - Window (all on `queue`) + + /// Rebuilds inFlight for one upload from the daemon's live tasks, and then + /// refills. A task in the .completed or .canceling state is NOT live: its + /// delegate callback, replayed after a relaunch, settles it. A part with no + /// live task simply enqueues again. Accept evaluation absorbs a + /// completed-but-unreported duplicate. We never guess. + /// `completion` fires, on `queue`, when this reconcile has settled: the + /// refill ran, or a newer reconcile superseded this one. reconcileAll gates + /// the background completion handlers on it. + private func reconcileLocked(_ id: String, resumedByStart: Bool, + completion: (() -> Void)? = nil) { + let token = UUID() + reconcileToken[id] = token + enumerateAllTasks { tasks in + self.queue.async { + defer { completion?() } + guard self.reconcileToken[id] == token else { return } // superseded + let manifest = self.latest(id) + var live: [Int: String] = [:] + for (session, task) in tasks { + guard let ref = Self.partRef(session, task), ref.id == id, + task.state == .running || task.state == .suspended else { continue } + if ref.incarnation != manifest?.incarnation || live[ref.part] != nil { + // Never adopt a task from a replaced incarnation. Its bytes and + // URL belong to the old plan, and the token check in + // handlePartCompletion drops its late completion. Never adopt a + // second live task for one part index: concurrent PUTs of one + // partNum are verified unsafe on the server side. + task.cancel() + } else { + live[ref.part] = TaskMap.key(session, task) + } + } + self.inFlight[id] = live + self.reconcileToken[id] = nil + if let manifest { + // Temp files for accepted parts with no live task are orphans. + for index in manifest.parts.indices + where manifest.parts[index].accepted && live[index] == nil { + ChunkedStore.removePartFile(id, index) + } + } + self.refill(id, resumedByStart: resumedByStart) + } + } + } + + /// Fills the window back up to [ChunkedEngine.window] enqueued part tasks. + /// Called after every part completion (the background-wake refill that the + /// design's liveness rationale requires), after a retry cooldown, and at + /// the end of every reconcile. + private func refill(_ id: String, resumedByStart: Bool = false) { + guard reconcileToken[id] == nil, let manifest = latest(id) else { return } + // Stalled wins, even over all-accepted. The journaled terminal stands + // until an explicit startUpload resume. The resume clears the stall, + // lands here again, and completes without a re-send. + guard !manifest.stalled else { return } + if manifest.allAccepted { + finalizeCompleted(id, manifest, reemit: resumedByStart) + return + } + let now = nowMs() + if manifest.isExpired(now) { + stall(id, entry: expiredEntry(id)) + return + } + armExpiryCheck(id, expiresAt: manifest.expiresAt) + // A blob shorter than a part's range can never finish. Report a terminal + // 'file' now, not a surprise when the window reaches the short part + // later. A retry cannot help, because the bytes are not there. Thus this + // stalls, and awaits removeUpload or a recreate whose tiling rule fits + // the real size. + let blobSize = ChunkedStore.blobSize(id) + if let short = manifest.parts.indices.first(where: { manifest.parts[$0].end > blobSize }) { + stall(id, entry: errorEntry( + id: id, + error: "source blob is \(blobSize) bytes; part \(short) needs " + + "[\(manifest.parts[short].start), \(manifest.parts[short].end))", + errorKind: "file", partIndex: short)) + return + } + let flight = Set((inFlight[id] ?? [:]).keys) + let cooling = Set((cooldownUntil[id] ?? [:]).filter { $0.value > now }.keys) + for index in ChunkedEngine.indexesToEnqueue( + pending: manifest.pendingIndexes(), inFlight: flight, cooling: cooling) { + if !enqueuePart(id, index, manifest) { return } // stalled inside + } + } + + private func enqueuePart(_ id: String, _ index: Int, _ manifest: ChunkedManifest) -> Bool { + let part = manifest.parts[index] + guard let url = URL(string: part.url) else { + stall(id, entry: errorEntry( + id: id, error: "part \(index) url is not a valid URL", errorKind: "unknown", + partIndex: index)) + return false + } + // A background session can upload only from a file. Thus each enqueued + // part gets a temp file that holds exactly its byte range. The transient + // disk usage stays at window × partSize, not a second full copy of the + // source. + let partFile: URL + do { + partFile = try ChunkedStore.writePartFile( + id: id, index: index, start: part.start, end: part.end, + incarnation: manifest.incarnation) + } catch { + stall(id, entry: errorEntry( + id: id, error: "cannot materialize part \(index): \(error.localizedDescription)", + errorKind: "file", partIndex: index)) + return false + } + var request = URLRequest(url: url) + request.httpMethod = "PUT" + // Unchanged, per the protocol-as-data rule. The library adds nothing. + for (key, value) in part.headers { + request.setValue(value, forHTTPHeaderField: key) + } + let session = uploader.session(wifiOnly: manifest.wifiOnly) + let task = session.uploadTask(with: request, fromFile: partFile) + task.taskDescription = ChunkedEngine.taskDescription( + id: id, part: index, incarnation: manifest.incarnation) + let key = TaskMap.key(session, task) + TaskMap.set(TaskMap.Meta(id: id, accept: nil, partIndex: index, + incarnation: manifest.incarnation), forKey: key) + inFlight[id, default: [:]][index] = key + task.resume() + return true + } + + // MARK: - Terminal transitions (all on `queue`) + + /// Journals the terminal, marks the upload stalled, and cancels its + /// in-flight tasks. The stall is durable: relaunch reconciliation must not + /// resume the upload; only startUpload may. The manifest and the bytes are + /// kept. Every non-completed terminal leaves the consumer its recovery + /// options. + private func stall(_ id: String, entry: JournaledEvent) { + _ = updateManifest(id) { manifest in + var next = manifest + next.stalled = true + return next + } + cancelTasks(for: id) + partSent[id] = nil + cooldownUntil[id] = nil + RNBackgroundUpload.journalAndEmit(entry) + } + + private func finalizeCompleted(_ id: String, _ manifest: ChunkedManifest, reemit: Bool) { + for index in manifest.parts.indices { ChunkedStore.removePartFile(id, index) } + partSent[id] = nil + // A resume of a finished-but-unacked upload must not mint a second + // terminal event. Emit the journaled event again. Thus a live listener + // still hears it, with the eventId that the consumer will ack. + if let existing = EventJournal.unacknowledgedEntries() + .first(where: { $0.id == id && $0.type == "completed" }) { + if reemit { RNBackgroundUpload.emitEvent(existing) } + return + } + // There are no response fields, because no single response represents N + // accepted parts. The blob is deleted only when this event is ACKED (see + // ackEvents). + RNBackgroundUpload.journalAndEmit( + JournaledEvent(eventId: UUID().uuidString, id: id, type: "completed", timestamp: nowMs())) + } + + // MARK: - Retry scheduling (all on `queue`) + + /// Counts one non-transient HTTP rejection against the budget of `part`. + /// The count lives in the manifest, persisted best-effort like the accepted + /// flag. Thus it survives process death and can trip across wakes. A resume + /// or a recreate resets it (ChunkedManifest.reconciled rebuilds the parts + /// from the incoming call). Over budget: journal the terminal 'http' and + /// stall. In budget: schedule the backoff retry when this callback owns the + /// part. For an unowned replay, the reconcile already in flight does the + /// re-enqueueing. + private func recordRejection(_ id: String, part: Int, manifest: ChunkedManifest, + code: Int, headers: [String: String], body: String?, + scheduleRetryInBudget: Bool) { + let count = (manifest.parts[part].rejections ?? 0) + 1 + _ = updateManifest(id) { $0.withPartRejections(part, count) } + if count > ChunkedEngine.partHttpRetries { + let (capped, truncated) = EventJournal.capBody(body) + var entry = errorEntry( + id: id, error: "HTTP \(code) on part \(part)", errorKind: "http", partIndex: part) + entry.responseCode = code + entry.responseBody = capped + entry.responseBodyTruncated = truncated + entry.responseHeaders = headers + stall(id, entry: entry) + } else if scheduleRetryInBudget { + scheduleRetry(id, part: part, attempt: count) + } + } + + private func scheduleTransientRetry(_ id: String, part: Int) { + let attempt = (transientAttempts[id]?[part] ?? 0) + 1 + transientAttempts[id, default: [:]][part] = attempt + scheduleRetry(id, part: part, attempt: attempt) + } + + private func scheduleRetry(_ id: String, part: Int, attempt: Int) { + let delayMs = ChunkedEngine.backoffMs(attempt: attempt) + cooldownUntil[id, default: [:]][part] = nowMs() + Double(delayMs) + queue.asyncAfter(deadline: .now() + .milliseconds(delayMs)) { [weak self] in + guard let self else { return } + self.cooldownUntil[id]?[part] = nil + self.refill(id) + } + } + + // Expiry is evaluated on every transition. But an upload whose tasks all + // wait (for connectivity, or for backoff) would pass its deadline silently + // while the app is alive. Thus we arm one timer at the deadline. When a + // resume extended expiresAt, the stale timer's refill is a no-op that arms + // the timer again. + private func armExpiryCheck(_ id: String, expiresAt: Double) { + guard !expiryArmed.contains(id) else { return } + expiryArmed.insert(id) + let delayMs = Int(min(max(expiresAt - nowMs(), 0) + 100, 7 * 24 * 3_600_000)) + queue.asyncAfter(deadline: .now() + .milliseconds(delayMs)) { [weak self] in + guard let self else { return } + self.expiryArmed.remove(id) + self.refill(id) + } + } + + // MARK: - Helpers + + // The stored copy is the truth. A reconcile can have replaced the headers + // or expiresAt. The cache exists for the progress path, and as a fallback + // when a read fails in flight. + private func latest(_ id: String) -> ChunkedManifest? { + guard let manifest = ChunkedStore.load(id) else { + manifests[id] = nil + return nil + } + manifests[id] = manifest + return manifest + } + + private func updateManifest( + _ id: String, _ transform: (ChunkedManifest) -> ChunkedManifest + ) -> ChunkedManifest? { + // Here the save is best-effort, unlike in startUpload. A lost accepted + // flag only causes a re-send of a part, and the server absorbs the + // duplicate through the accept rules. That is better than a failed upload + // that the server in fact took. + let next = ChunkedStore.update(id, transform) ?? manifests[id].map(transform) + if let next { manifests[id] = next } + return next + } + + private func takeOwnership(path: String, id: String) throws { + let source = URL(string: path) ?? URL(fileURLWithPath: path) + let blob = ChunkedStore.blobURL(id) + let fm = FileManager.default + guard fm.fileExists(atPath: source.path) else { + // A crash between the move and the manifest save leaves the bytes at + // the blob path with no manifest. Adopt the bytes. Do not fail the + // retry. + if fm.fileExists(atPath: blob.path) { return } + throw ChunkedManifest.ParseError( + message: "chunked source file does not exist: \(source.path)") + } + try fm.createDirectory(at: ChunkedStore.uploadDir(id), withIntermediateDirectories: true) + try? fm.removeItem(at: blob) + // This is an O(1) rename on the same volume. Across volumes, FileManager + // falls back to a copy. + try fm.moveItem(at: source, to: blob) + } + + private func emitAggregateProgress(_ id: String, _ manifest: ChunkedManifest) { + let total = manifest.totalBytes + guard total > 0 else { return } + let sent = min(manifest.acceptedBytes + (partSent[id]?.values.reduce(0, +) ?? 0), total) + RNBackgroundUpload.emitProgress(id: id, progress: 100.0 * Float(sent) / Float(total)) + } + + private func clearState(_ id: String) { + inFlight[id] = nil + reconcileToken[id] = nil // discards any pending reconcile snapshot + partSent[id] = nil + cooldownUntil[id] = nil + transientAttempts[id] = nil + manifests[id] = nil + // A removed-then-recreated id must be able to arm its own expiry + // deadline, which is possibly earlier. It must not wait out the stale + // timer. + expiryArmed.remove(id) + progressLock.lock() + lastProgressAt[id] = nil // without this, one entry per id stays forever + progressLock.unlock() + } + + private func cancelTasks(for id: String) { + enumerateAllTasks { tasks in + for (session, task) in tasks where Self.partRef(session, task)?.id == id { + task.cancel() + } + } + } + + // Always examine both sessions. A resume can change wifiOnly while earlier + // part tasks continue where they started. + private func enumerateAllTasks( + _ completion: @escaping ([(URLSession, URLSessionTask)]) -> Void + ) { + let sessions = [uploader.session(wifiOnly: false), uploader.session(wifiOnly: true)] + let group = DispatchGroup() + let lock = NSLock() + var collected: [(URLSession, URLSessionTask)] = [] + for session in sessions { + group.enter() + session.getAllTasks { tasks in + lock.lock() + collected.append(contentsOf: tasks.map { (session, $0) }) + lock.unlock() + group.leave() + } + } + group.notify(queue: .global()) { completion(collected) } + } + + private func expiredEntry(_ id: String) -> JournaledEvent { + errorEntry(id: id, error: "upload expired before every part was accepted", + errorKind: "expired") + } + + private func errorEntry(id: String, error: String, errorKind: String, + partIndex: Int? = nil) -> JournaledEvent { + var entry = JournaledEvent( + eventId: UUID().uuidString, id: id, type: "error", timestamp: nowMs()) + entry.error = error + entry.errorKind = errorKind + entry.partIndex = partIndex + return entry + } +} diff --git a/ios/ChunkedEngine.swift b/ios/ChunkedEngine.swift new file mode 100644 index 00000000..caf3007c --- /dev/null +++ b/ios/ChunkedEngine.swift @@ -0,0 +1,81 @@ +import Foundation + +// The pure scheduling half of chunked execution: window arithmetic, the +// retry policy, backoff, and the part-task identity encoding. It is kept free +// of session state. Thus the highest-consequence invariants (at most WINDOW +// part tasks enqueued per upload, and never two for one part index) can be +// examined in one place. [ChunkedCoordinator] owns the session side. +enum ChunkedEngine { + + // The number of part tasks of one upload enqueued with the daemon at one + // time. It is a library constant, not an option: if soak data argues for a + // different value, this constant changes, not the API. The window is also a + // liveness decision. A background session only progresses tasks that are + // already enqueued. Thus WINDOW tasks of runway let a multi-part upload + // proceed while the app is dead. Without them, the upload pays a + // rate-limited wake per part. + static let window = 3 + + // A non-accepted, non-transient HTTP response is retried this many times + // for each part. Then it becomes a terminal error and stalls the upload. + // The number is small on purpose. A response that the server repeats (401, + // 400) will not change without a new startUpload. Only transient failures + // retry without a limit. + static let partHttpRetries = 3 + + private static let backoffBaseMs = 1_000 + private static let backoffCapMs = 60_000 + + // A 5xx means that the server fails, not that the request is wrong. Thus + // it retries like a transport failure: without a limit, within expiresAt. + static func isTransientHttp(_ code: Int) -> Bool { (500...599).contains(code) } + + /// Exponential backoff for transient failures: 1s, 2s, 4s, up to a 60s cap. + static func backoffMs(attempt: Int) -> Int { + min(backoffBaseMs << min(max(attempt - 1, 0), 6), backoffCapMs) + } + + /// The part indexes to enqueue now: pending (not accepted), not already + /// enqueued, and not cooling down after a failure, up to the window size. + /// It never returns an index in `inFlight`. That is the one-task-per-part + /// invariant. + static func indexesToEnqueue( + pending: [Int], inFlight: Set, cooling: Set, window: Int = window + ) -> [Int] { + let slots = window - inFlight.count + guard slots > 0 else { return [] } + return Array(pending.filter { !inFlight.contains($0) && !cooling.contains($0) }.prefix(slots)) + } + + // MARK: - Part-task identity + + // A chunked part task must carry (uploadId, partIndex, incarnation) + // through the daemon. taskDescription is the primary carrier. It is a + // prefix plus JSON, so a consumer id that contains a delimiter survives. + // TaskMap holds the same triple as the durable fallback, per the DTS + // guidance that TaskMap documents. The incarnation is the manifest token + // that the task was created under. A callback whose token no longer matches + // the stored manifest's token is from a removed or replaced plan. It must + // not write into the current plan. + private static let descriptionPrefix = "rnbgu-chunk:" + + private struct PartRef: Codable { + let id: String + let part: Int + var inc: String? + } + + static func taskDescription(id: String, part: Int, incarnation: String) -> String { + let data = (try? JSONEncoder().encode(PartRef(id: id, part: part, inc: incarnation))) ?? Data() + return descriptionPrefix + (String(data: data, encoding: .utf8) ?? "") + } + + static func parseTaskDescription( + _ description: String? + ) -> (id: String, part: Int, incarnation: String?)? { + guard let description, description.hasPrefix(descriptionPrefix) else { return nil } + let json = Data(description.dropFirst(descriptionPrefix.count).utf8) + guard let ref = try? JSONDecoder().decode(PartRef.self, from: json) else { return nil } + return (ref.id, ref.part, ref.inc) + } +} diff --git a/ios/ChunkedManifest.swift b/ios/ChunkedManifest.swift new file mode 100644 index 00000000..5375cd3c --- /dev/null +++ b/ios/ChunkedManifest.swift @@ -0,0 +1,404 @@ +import Foundation + +/// The durable record of one chunked upload: the parts that the consumer +/// authored, and which of them the server has accepted. [ChunkedStore] saves +/// it at startUpload, BEFORE any task is enqueued. Thus a process that the +/// system relaunches (or a startUpload after a crash, a stop, or a reauth) +/// resumes from it without a call into JS. This manifest IS the resume +/// mechanism. +/// +/// The content is the same as the Android manifest, with two platform +/// differences: +/// - There is no sourcePath field. iOS moves the app container between +/// launches, so an absolute path would go stale. The moved bytes live at a +/// location derived from the id (ChunkedStore.blobURL). +/// - `stalled` is persisted. On Android, "stalled" only means that the worker +/// is not scheduled. iOS reconciles every upload on relaunch. Thus an +/// upload that journaled a terminal outcome needs a durable marker that +/// says: await an explicit startUpload resume, and do not refill. +struct ChunkedManifest: Codable { + /// One part, exactly as the consumer authored it. The library sends the + /// file bytes [start, end) as the body of a PUT to `url`, with `headers` + /// unchanged. It never derives or edits a protocol field. + struct Part: Codable { + let url: String + var headers: [String: String] + let start: Int64 + let end: Int64 // exclusive + var accepted: Bool = false + /// The non-transient HTTP rejections counted against this part's retry + /// budget (nil means 0). It is persisted so that the budget survives + /// process death. An in-memory count resets on every system wake. That + /// would let a deterministic 4xx upload the part again until expiresAt, + /// with no terminal ever journaled. The count resets when the part is + /// rebuilt from an incoming call. Resume and recreate both do that (see + /// [reconciled]). + var rejections: Int? + + var size: Int64 { end - start } + } + + let id: String + var parts: [Part] + var accept: [UploadOutcome.AcceptRule] + /// Epoch ms. After this time, the upload stops with errorKind 'expired'. + var expiresAt: Double + var wifiOnly: Bool + let createdAt: Double + var stalled: Bool = false + /// The identity of this parts plan. It rotates on a recreate (a startUpload + /// that replaced the parts wholesale). It never rotates on a resume. Part + /// tasks carry it in their identity. Thus a late delegate callback from a + /// removed or replaced incarnation can be told apart from the live plan's + /// callbacks and dropped. Its response is about byte ranges and URLs that + /// this manifest no longer describes. + var incarnation: String + + var totalBytes: Int64 { parts.reduce(0) { $0 + $1.size } } + var acceptedBytes: Int64 { parts.filter(\.accepted).reduce(0) { $0 + $1.size } } + + /// The server's auto-publish condition. It is the only thing that + /// 'completed' may mean. + var allAccepted: Bool { parts.allSatisfy(\.accepted) } + + func isExpired(_ nowMs: Double) -> Bool { nowMs >= expiresAt } + + func pendingIndexes() -> [Int] { parts.indices.filter { !parts[$0].accepted } } + + func withPartAccepted(_ index: Int) -> ChunkedManifest { + var next = self + next.parts[index].accepted = true + return next + } + + func withPartRejections(_ index: Int, _ count: Int) -> ChunkedManifest { + var next = self + next.parts[index].rejections = count + return next + } + + struct ReconcileError: LocalizedError { + let message: String + var errorDescription: String? { message } + } + + /// A new startUpload call with an existing id is one of two things. The + /// semantics are identical to Android's `ChunkedManifest.reconcile`. + /// + /// **Resume** — the incoming parts are the SAME array (identical count, + /// ranges, and urls). The headers, the accept rules, expiresAt, and wifiOnly + /// come from the new call. This is how fresh auth reaches stalled parts, + /// and how a salvage extends the deadline. The accepted part statuses, + /// createdAt, and the incarnation survive from this manifest. A resume is + /// permitted at any time, running or not. The stall clears, because a + /// resume is the whole point of the new call. + /// + /// **Recreate** — a DIFFERENT parts array: the consumer authored the upload + /// again, under a fresh server uploadId, after the old one died. The owned + /// bytes are kept. The parts are replaced wholesale. Every part status + /// resets to unsent. The headers, accept rules, and expiresAt come from the + /// new call. The new ranges must tile exactly [0, blobSize). A partial or + /// overlapping cover would silently upload wrong bytes. A recreate is + /// accepted only while the upload is NOT running (stalled on a terminal + /// error or cancel, or expired). A different parts array while part tasks + /// are live is a consumer bug, not a recreate, because the in-flight + /// requests belong to the old parts. The incarnation rotates to the + /// incoming manifest's fresh token. Thus late callbacks from the replaced + /// parts are dropped. + func reconciled(with incoming: ChunkedManifest, running: Bool, + blobSize: Int64) throws -> ChunkedManifest { + if samePartsAs(incoming) { + // Built from `incoming`, so the per-part rejection counts reset. A + // resume arrives with fresh headers and gets a fresh retry budget. + let mergedParts = incoming.parts.enumerated().map { i, new -> Part in + var part = new + part.accepted = parts[i].accepted + return part + } + return ChunkedManifest( + id: id, parts: mergedParts, accept: incoming.accept, expiresAt: incoming.expiresAt, + wifiOnly: incoming.wifiOnly, createdAt: createdAt, stalled: false, + incarnation: incarnation) + } + guard !running else { + throw ReconcileError(message: + "chunked upload '\(id)' is running; a different parts array is only accepted once it stops") + } + guard Self.tilesExactly(incoming.parts, size: blobSize) else { + throw ReconcileError(message: + "chunked upload '\(id)' recreate parts must tile exactly [0, \(blobSize))") + } + return ChunkedManifest( + id: id, parts: incoming.parts, accept: incoming.accept, expiresAt: incoming.expiresAt, + wifiOnly: incoming.wifiOnly, createdAt: createdAt, stalled: false, + incarnation: incoming.incarnation) + } + + private func samePartsAs(_ incoming: ChunkedManifest) -> Bool { + incoming.parts.count == parts.count && parts.indices.allSatisfy { i in + incoming.parts[i].url == parts[i].url + && incoming.parts[i].start == parts[i].start + && incoming.parts[i].end == parts[i].end + } + } + + /// Tells whether `parts` cover [0, size) exactly: no gap, no overlap, and + /// nothing past the end. It is order-independent, like everything else + /// about parts. + static func tilesExactly(_ parts: [Part], size: Int64) -> Bool { + guard !parts.isEmpty else { return false } + var cursor: Int64 = 0 + for part in parts.sorted(by: { $0.start < $1.start }) { + guard part.start == cursor, part.end > part.start else { return false } + cursor = part.end + } + return cursor == size + } + + struct ParseError: LocalizedError { + let message: String + var errorDescription: String? { message } + } + + /// Turns bridged options into a manifest. It throws on each field that the + /// engine relies on. JS validates first. Thus a throw here is a bug worth + /// surfacing, not UX. + static func parse(_ options: [String: Any], createdAt: Double) throws -> ChunkedManifest { + guard let id = options["id"] as? String, !id.isEmpty else { + throw ParseError(message: "Missing 'id'") + } + guard let rawParts = options["parts"] as? [[String: Any]], !rawParts.isEmpty else { + throw ParseError(message: "'parts' must be a non-empty array") + } + guard let expiresAt = (options["expiresAt"] as? NSNumber)?.doubleValue else { + throw ParseError(message: "Missing 'expiresAt'") + } + let parts = try rawParts.enumerated().map { i, raw -> Part in + guard let url = raw["url"] as? String else { + throw ParseError(message: "Missing 'parts[\(i)].url'") + } + guard let range = raw["range"] as? [String: Any], + let start = (range["start"] as? NSNumber)?.int64Value, + let end = (range["end"] as? NSNumber)?.int64Value, + start >= 0, start < end else { + throw ParseError(message: "Invalid 'parts[\(i)].range'") + } + return Part(url: url, headers: parseHeaders(raw["headers"]), start: start, end: end) + } + return ChunkedManifest( + id: id, + parts: parts, + accept: UploadOutcome.parseAcceptRules(options["accept"]), + expiresAt: expiresAt, + wifiOnly: (options["wifiOnly"] as? Bool) ?? false, + createdAt: createdAt, + incarnation: UUID().uuidString) + } + + // The same header coercion as the simple-upload path: strings and numbers + // only. Anything else is skipped. It is not interpolated onto the wire. + private static func parseHeaders(_ raw: Any?) -> [String: String] { + guard let headers = raw as? [String: Any] else { return [:] } + var result: [String: String] = [:] + for (key, value) in headers { + if let s = value as? String { + result[key] = s + } else if let n = value as? NSNumber { + result[key] = n.stringValue + } + } + return result + } +} + +/// A file-backed store: one directory per upload id. The directory holds +/// `manifest.json`, `blob` (the moved source bytes), and the in-flight part +/// temp files. It has the same durability pattern as [EventJournal]: a +/// synchronous serial queue, atomic writes, and corrupt files read as +/// absent. +enum ChunkedStore { + struct StoreError: LocalizedError { + let message: String + var errorDescription: String? { message } + } + + private static let queue = DispatchQueue(label: "ai.openspace.rnbgupload.chunkedstore") + + private static let dirURL: URL = { + let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] + var dir = base.appendingPathComponent("RNFileUploaderChunked", isDirectory: true) + try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) + // This is device-local upload state. Keep it out of iCloud/iTunes + // backups. + var values = URLResourceValues() + values.isExcludedFromBackup = true + try? dir.setResourceValues(values) + return dir + }() + + // Upload ids come from the consumer. They can contain path separators or + // other filesystem-hostile characters. Thus the directory name is an + // encoding of the id, never the id itself. The id is read back from the + // manifest, not decoded from the name. + static func uploadDir(_ id: String) -> URL { + dirURL.appendingPathComponent( + Data(id.utf8).base64EncodedString() + .replacingOccurrences(of: "+", with: "-") + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "=", with: ""), + isDirectory: true) + } + + private static func manifestURL(_ id: String) -> URL { + uploadDir(id).appendingPathComponent("manifest.json") + } + + /// The location where startUpload moves the source file for this id. + static func blobURL(_ id: String) -> URL { + uploadDir(id).appendingPathComponent("blob") + } + + /// The temp file that holds exactly the byte range of part `index` while + /// that part is enqueued with the daemon (a background session can upload + /// only from a file). The name encodes the manifest incarnation and the + /// byte range. Thus a stale file from a previous incarnation or a different + /// plan can never be adopted by a size coincidence. Reuse checks the full + /// identity, not only the byte count. + static func partFileURL(_ id: String, _ index: Int, incarnation: String, + start: Int64, end: Int64) -> URL { + uploadDir(id).appendingPathComponent("part-\(index).\(incarnation).\(start)-\(end)") + } + + /// The size in bytes of the moved blob. It is 0 when the blob is missing. + static func blobSize(_ id: String) -> Int64 { + queue.sync { + (((try? FileManager.default.attributesOfItem(atPath: blobURL(id).path))?[.size] + as? NSNumber)?.int64Value) ?? 0 + } + } + + static func load(_ id: String) -> ChunkedManifest? { + queue.sync { read(manifestURL(id)) } + } + + /// Throws on a write failure. A manifest that did not persist must fail + /// the startUpload call. + static func save(_ manifest: ChunkedManifest) throws { + try queue.sync { + try FileManager.default.createDirectory( + at: uploadDir(manifest.id), withIntermediateDirectories: true) + let data = try JSONEncoder().encode(manifest) + try data.write(to: manifestURL(manifest.id), options: .atomic) + } + } + + /// An atomic read-modify-write. Thus a mark of one part as accepted can + /// never clobber a concurrent reconcile's fresh headers, or another part's + /// flag. It returns nil, and does not throw, when the manifest is gone or + /// the write failed. Callers that can proceed from memory do so. + static func update(_ id: String, _ transform: (ChunkedManifest) -> ChunkedManifest) -> ChunkedManifest? { + queue.sync { + guard let manifest = read(manifestURL(id)) else { return nil } + let next = transform(manifest) + guard let data = try? JSONEncoder().encode(next) else { return nil } + do { + try data.write(to: manifestURL(id), options: .atomic) + return next + } catch { + return nil + } + } + } + + /// Deletes the manifest, the moved bytes, AND all part temp files. It does + /// nothing for an unknown id (for example, a simple upload's id). + static func remove(_ id: String) { + queue.sync { try? FileManager.default.removeItem(at: uploadDir(id)) } + } + + static func all() -> [ChunkedManifest] { + queue.sync { + let dirs = (try? FileManager.default.contentsOfDirectory( + at: dirURL, includingPropertiesForKeys: nil)) ?? [] + return dirs.compactMap { read($0.appendingPathComponent("manifest.json")) } + } + } + + // Codable enforces the non-optional fields at decode time, unlike Gson. + // Thus a corrupt or field-renamed file simply reads as absent. + private static func read(_ url: URL) -> ChunkedManifest? { + guard let data = try? Data(contentsOf: url) else { return nil } + guard let m = try? JSONDecoder().decode(ChunkedManifest.self, from: data), + !m.parts.isEmpty else { return nil } + return m + } + + /// Writes bytes [start, end) of the blob into `dest`. It writes a tmp file + /// and renames it. Thus a partial write can never be mistaken for a + /// finished part file. It throws when the blob is missing or shorter than + /// `end`. For the caller that is a 'file' terminal, because a retry can + /// never succeed. + static func writePartFile(id: String, index: Int, start: Int64, end: Int64, + incarnation: String) throws -> URL { + try queue.sync { + let dest = partFileURL(id, index, incarnation: incarnation, start: start, end: end) + // Sweep the other files of this index first. A temp file left by a + // replaced incarnation or plan must not stay and leak disk. It cannot + // be reused, because the identity is in the name, but it can pile up. + removePartFilesLocked(id, index, keeping: dest) + // An existing file with exactly this identity and size is a finished + // copy from a previous enqueue of this part. Reuse it. Size alone is + // not trusted. The name carries the incarnation and the range that + // produced the file. + if let size = try? FileManager.default.attributesOfItem(atPath: dest.path)[.size] as? NSNumber, + size.int64Value == end - start { + return dest + } + let blob = blobURL(id) + let blobSize = ((try FileManager.default.attributesOfItem(atPath: blob.path)[.size] + as? NSNumber)?.int64Value) ?? 0 + guard blobSize >= end else { + throw StoreError( + message: "source blob is \(blobSize) bytes; part \(index) needs [\(start), \(end))") + } + let tmp = uploadDir(id).appendingPathComponent("part-\(index).tmp") + FileManager.default.createFile(atPath: tmp.path, contents: nil) + let reader = try FileHandle(forReadingFrom: blob) + defer { try? reader.close() } + let writer = try FileHandle(forWritingTo: tmp) + defer { try? writer.close() } + try reader.seek(toOffset: UInt64(start)) + var remaining = end - start + while remaining > 0 { + let chunk = Int(min(remaining, 1 << 20)) + guard let data = try reader.read(upToCount: chunk), !data.isEmpty else { + throw StoreError(message: "short read building part \(index)") + } + try writer.write(contentsOf: data) + remaining -= Int64(data.count) + } + try? FileManager.default.removeItem(at: dest) + try FileManager.default.moveItem(at: tmp, to: dest) + return dest + } + } + + /// Removes every file for part `index`: the current incarnation's file, + /// stale files, and half-written tmp files. They all share the + /// `part-.` prefix. + static func removePartFile(_ id: String, _ index: Int) { + queue.sync { removePartFilesLocked(id, index, keeping: nil) } + } + + // Must run on `queue`. The `part-.` prefix cannot collide across + // indexes ("part-1." is not a prefix of "part-12.<...>"). + private static func removePartFilesLocked(_ id: String, _ index: Int, keeping: URL?) { + let files = (try? FileManager.default.contentsOfDirectory( + at: uploadDir(id), includingPropertiesForKeys: nil)) ?? [] + for file in files + where file.lastPathComponent.hasPrefix("part-\(index).") + && file.lastPathComponent != keeping?.lastPathComponent { + try? FileManager.default.removeItem(at: file) + } + } +} diff --git a/ios/EventJournal.swift b/ios/EventJournal.swift index 9f3f2d24..3011acd8 100644 --- a/ios/EventJournal.swift +++ b/ios/EventJournal.swift @@ -11,8 +11,9 @@ struct JournaledEvent: Codable { var responseBodyTruncated: Bool? var responseHeaders: [String: String]? var error: String? - var errorKind: String? // http | network | file | unknown + var errorKind: String? // http | network | file | expired | unknown var cancelReason: String? // user | system + var partIndex: Int? // chunked uploads only: the failing part, when known // Bridge-friendly dictionary (nil fields omitted so nothing becomes NSNull). var bridged: [String: Any] { @@ -24,6 +25,7 @@ struct JournaledEvent: Codable { if let error { m["error"] = error } if let errorKind { m["errorKind"] = errorKind } if let cancelReason { m["cancelReason"] = cancelReason } + if let partIndex { m["partIndex"] = partIndex } return m } } @@ -88,6 +90,10 @@ enum EventJournal { } static func unacknowledged() -> [[String: Any]] { + unacknowledgedEntries().map { $0.bridged } + } + + static func unacknowledgedEntries() -> [JournaledEvent] { queue.sync { let files = (try? FileManager.default.contentsOfDirectory(at: dirURL, includingPropertiesForKeys: nil)) ?? [] return files @@ -97,7 +103,6 @@ enum EventJournal { return try? JSONDecoder().decode(JournaledEvent.self, from: data) } .sorted { $0.timestamp < $1.timestamp } - .map { $0.bridged } } } diff --git a/ios/RNBackgroundUpload.swift b/ios/RNBackgroundUpload.swift index 7a11480f..ca16e166 100644 --- a/ios/RNBackgroundUpload.swift +++ b/ios/RNBackgroundUpload.swift @@ -44,6 +44,19 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { private static var responsesData: [String: NSMutableData] = [:] // sessionId:taskId -> body private static var lastProgressAt: [String: TimeInterval] = [:] // uploadId -> time private static var userCancelledIds = Set() + // The ids that removeUpload is releasing now. The cancellation of their + // tasks is an explicit release, not an outcome that the consumer awaits. + // Thus no terminal event is journaled. This matches Android, whose + // removeUpload cancels work with no user-cancel mark. + private static var removedIds = Set() + // The consumer-supplied ids whose check-and-create is in flight, mapped to + // the resolves of the concurrent same-id calls. The existence check + // enumerates the session tasks asynchronously. Without this claim, two + // concurrent calls could both see "no task" and enqueue duplicates. The id + // is claimed synchronously, under `lock`, BEFORE the enumeration is + // dispatched. The map entry drains when the first caller's create-or-find + // lands. + private static var creationsInFlight: [String: [RCTPromiseResolveBlock]] = [:] private static var backgroundSession: URLSession? private static var wifiOnlySession: URLSession? @@ -58,6 +71,19 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { // id) so the app can be relaunched to finish uploads after termination. private static let bgHandlerLock = NSLock() private static var bgCompletionHandlers: [String: () -> Void] = [:] + // Relaunch ordering: while the chunked coordinator reconciles (deferrals + // > 0), urlSessionDidFinishEvents must NOT hand the system its completion + // handler. The system could suspend the app before the post-reconcile + // refill enqueues a new part task. That would leave zero daemon tasks and + // no future wake. A session that finishes its events in that window parks + // its id here. The release drains it. + private static var bgHandlerDeferrals = 0 + private static var bgSessionsAwaitingDrain: Set = [] + + // Owns the chunked-upload window and the manifests. It is implicitly + // unwrapped only because it needs `self` (for the sessions) and is assigned + // before init returns. It is never nil after that. + private var chunked: ChunkedCoordinator! public override init() { super.init() @@ -65,6 +91,12 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { // nsurlsessiond from a previous launch are delivered to this process. _ = session(wifiOnly: false) _ = session(wifiOnly: true) + chunked = ChunkedCoordinator(uploader: self) + // Relaunch reconciliation: match the daemon's surviving tasks against + // the stored manifests, and refill each upload's window. It runs on the + // coordinator queue. Thus nothing here re-enters the initialization of + // `shared`. + chunked.reconcileAll() } // MARK: - Event delegate @@ -97,9 +129,35 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { return eventDelegate } + // Journal-before-emit, the library's one terminal-event path. The write is + // durable. The emit is best-effort, because JS can be dead. The + // simple-upload delegate handling and the chunked coordinator share it. + static func journalAndEmit(_ event: JournaledEvent) { + EventJournal.append(event) + emitEvent(event) + } + + /// Emits WITHOUT a journal write. Use it to deliver again an event that is + /// already in the journal (a resume of a finished-but-unacked upload). + static func emitEvent(_ event: JournaledEvent) { + let body = event.bridged + let delegate = currentDelegate + switch event.type { + case "completed": delegate?.emitCompleted(body) + case "cancelled": delegate?.emitCancelled(body) + default: delegate?.emitError(body) + } + } + + static func emitProgress(id: String, progress: Float) { + currentDelegate?.emitProgress(["id": id, "progress": progress]) + } + // MARK: - Sessions - private func session(wifiOnly: Bool) -> URLSession { + // Internal, not private: the chunked coordinator enqueues part tasks on the + // same two sessions. + func session(wifiOnly: Bool) -> URLSession { RNBackgroundUpload.lock.lock() defer { RNBackgroundUpload.lock.unlock() } if wifiOnly { @@ -130,16 +188,19 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { } private func taskMapKey(_ session: URLSession, _ task: URLSessionTask) -> String { - "\(session.configuration.identifier ?? ""):\(task.taskIdentifier)" + TaskMap.key(session, task) } // taskDescription is the primary id; the persisted map is the durable fallback. + // A chunked part task's description encodes (uploadId, partIndex). This + // returns the uploadId in both cases. Thus id matching works uniformly. private func uploadId(_ session: URLSession, _ task: URLSessionTask) -> String { - task.taskDescription ?? TaskMap.meta(forKey: taskMapKey(session, task))?.id ?? "unknown" + if let ref = ChunkedCoordinator.partRef(session, task) { return ref.id } + return task.taskDescription ?? TaskMap.meta(forKey: taskMapKey(session, task))?.id ?? "unknown" } - private func acceptStatus(_ session: URLSession, _ task: URLSessionTask) -> [Int] { - TaskMap.meta(forKey: taskMapKey(session, task))?.acceptStatus ?? [] + private func acceptRules(_ session: URLSession, _ task: URLSessionTask) -> [UploadOutcome.AcceptRule] { + TaskMap.meta(forKey: taskMapKey(session, task))?.accept ?? [] } private var activeSessions: [URLSession] { @@ -180,9 +241,7 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { } let wifiOnly = (options["wifiOnly"] as? Bool) ?? false - // RN bridges a JS number[] to NSArray; map explicitly rather than - // rely on an [Int] bridging cast that can yield nil and silently drop it. - let acceptStatus = (options["acceptStatus"] as? [NSNumber])?.map { $0.intValue } ?? [] + let accept = UploadOutcome.parseAcceptRules(options["accept"]) let uploadId = (options["id"] as? String) ?? UUID().uuidString let fileURL = URL(string: path) ?? URL(fileURLWithPath: path) @@ -190,10 +249,9 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { let startNew = { let task = session.uploadTask(with: request, fromFile: fileURL) task.taskDescription = uploadId - TaskMap.set(TaskMap.Meta(id: uploadId, acceptStatus: acceptStatus), + TaskMap.set(TaskMap.Meta(id: uploadId, accept: accept, partIndex: nil), forKey: self.taskMapKey(session, task)) task.resume() - resolve(uploadId) } // A consumer-supplied id makes startUpload idempotent. This is the same @@ -203,7 +261,32 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { // different wifiOnly value while the first task continues in its first // session. A generated id cannot collide, so that path does not do the // (asynchronous) task enumeration. - guard options["id"] != nil else { startNew(); return } + guard options["id"] != nil else { + startNew() + resolve(uploadId) + return + } + + // Serialize the check-and-create for each id: claim the id synchronously, + // before we dispatch the enumeration. The first caller runs the check and + // creates the task. A concurrent same-id caller parks its resolve here. + // When the task lands, we answer the parked calls with the id. There is no + // second task, and there is no polling. + RNBackgroundUpload.lock.lock() + if RNBackgroundUpload.creationsInFlight[uploadId] != nil { + RNBackgroundUpload.creationsInFlight[uploadId]?.append(resolve) + RNBackgroundUpload.lock.unlock() + return + } + RNBackgroundUpload.creationsInFlight[uploadId] = [] + RNBackgroundUpload.lock.unlock() + let settle = { + RNBackgroundUpload.lock.lock() + let waiters = RNBackgroundUpload.creationsInFlight.removeValue(forKey: uploadId) ?? [] + RNBackgroundUpload.lock.unlock() + resolve(uploadId) + for waiter in waiters { waiter(uploadId) } + } let group = DispatchGroup() let foundLock = NSLock() @@ -222,7 +305,64 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { } } group.notify(queue: .main) { - if exists { resolve(uploadId) } else { startNew() } + if !exists { startNew() } + settle() + } + } + + @objc(startChunkedUpload:resolve:reject:) + public func startChunkedUpload(_ options: [String: Any], + resolve: @escaping RCTPromiseResolveBlock, + reject: @escaping RCTPromiseRejectBlock) { + chunked.startUpload( + options, + resolve: { id in resolve(id) }, + reject: { message in reject("RN Uploader", message, nil) }) + } + + @objc(removeUpload:resolve:reject:) + public func removeUpload(_ uploadId: String, + resolve: @escaping RCTPromiseResolveBlock, + reject: @escaping RCTPromiseRejectBlock) { + // The chunked release runs first: it cancels the in-flight part tasks + // and deletes the manifest and the bytes. Then we cancel any simple task + // that wears this id. That cancel is kept out of the journal, because an + // explicit release is not an outcome that the consumer awaits. + chunked.remove(uploadId) { + RNBackgroundUpload.lock.lock() + RNBackgroundUpload.removedIds.insert(uploadId) + RNBackgroundUpload.lock.unlock() + let group = DispatchGroup() + let foundLock = NSLock() + var found = false + for session in self.activeSessions { + group.enter() + session.getAllTasks { tasks in + for task in tasks + where self.uploadId(session, task) == uploadId + && ChunkedCoordinator.partRef(session, task) == nil { + foundLock.lock() + found = true + foundLock.unlock() + task.cancel() + } + group.leave() + } + } + group.notify(queue: .main) { + foundLock.lock() + let matched = found + foundLock.unlock() + if !matched { + // Nothing was cancelled. Thus no delegate callback will consume + // the suppression. Drop it. If we keep it, a later upload that + // reuses this id has its real terminal swallowed. + RNBackgroundUpload.lock.lock() + RNBackgroundUpload.removedIds.remove(uploadId) + RNBackgroundUpload.lock.unlock() + } + resolve(nil) + } } } @@ -230,6 +370,18 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { public func cancelUpload(_ cancelUploadId: String, resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { + // A chunked upload cancels through its coordinator: one 'cancelled' + // terminal for the whole upload, journaled before its part tasks are torn + // down. nil means that the id has no manifest. It then falls through to + // the simple-task path. + chunked.cancel(cancelUploadId) { handled in + if let handled { resolve(handled); return } + self.cancelSimpleUpload(cancelUploadId, resolve: resolve) + } + } + + private func cancelSimpleUpload(_ cancelUploadId: String, + resolve: @escaping RCTPromiseResolveBlock) { // Record intent before cancelling so the delegate reports cancelReason 'user'. RNBackgroundUpload.lock.lock() RNBackgroundUpload.userCancelledIds.insert(cancelUploadId) @@ -278,8 +430,16 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { public func ackEvents(_ eventIds: [String], resolve: @escaping RCTPromiseResolveBlock, reject: @escaping RCTPromiseRejectBlock) { + // An acked 'completed' is the ONE moment when a chunked upload's manifest + // and moved bytes may be deleted. Every other terminal keeps them for a + // resume. Find those uploads before the entries are removed. + let completedUploadIds = EventJournal.unacknowledgedEntries() + .filter { $0.type == "completed" && eventIds.contains($0.eventId) } + .map { $0.id } EventJournal.ack(eventIds) - resolve(true) + chunked.releaseCompleted(completedUploadIds) { // no-op for simple uploads + resolve(true) + } } @objc(getAllUploads:reject:) @@ -293,6 +453,9 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { group.enter() session.getAllTasks { tasks in for task in tasks { + // A chunked upload is one logical row, built from its manifest + // below. Its per-part tasks are transport detail. + if ChunkedCoordinator.partRef(session, task) != nil { continue } let id = self.uploadId(session, task) if id == "unknown" { continue } lock.lock() @@ -317,7 +480,14 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { group.leave() } } - group.notify(queue: .main) { resolve(result) } + group.notify(queue: .main) { + self.chunked.snapshots { chunkedRows in + lock.lock() + let combined = result + chunkedRows + lock.unlock() + resolve(combined) + } + } } // Called from AppDelegate.application(_:handleEventsForBackgroundURLSession:completionHandler:). @@ -329,13 +499,45 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { forIdentifier identifier: String) { // Touching `shared` recreates the background sessions when this is a fresh, // system-relaunched process, which is what lets the queued delegate events - // (and therefore this handler) actually fire. + // (and therefore this handler) actually fire. On a relaunch, it also + // claims the handler deferral (see below) BEFORE the handler is stored + // here. Thus the claim provably precedes any drain. _ = shared bgHandlerLock.lock() bgCompletionHandlers[identifier] = handler bgHandlerLock.unlock() } + /// For the chunked coordinator only. It parks every + /// urlSessionDidFinishEvents drain until the matching release. Thus the + /// system cannot suspend the app between a relaunch's replayed part + /// completions and the post-reconcile refill that enqueues the next part + /// tasks. + static func deferBackgroundCompletionHandlers() { + bgHandlerLock.lock() + bgHandlerDeferrals += 1 + bgHandlerLock.unlock() + } + + static func releaseBackgroundCompletionHandlers() { + bgHandlerLock.lock() + bgHandlerDeferrals -= 1 + var handlers: [() -> Void] = [] + if bgHandlerDeferrals <= 0 { + for identifier in bgSessionsAwaitingDrain { + if let handler = bgCompletionHandlers.removeValue(forKey: identifier) { + handlers.append(handler) + } + // A parked id with no stored handler is simply dropped. There is + // nothing to hold. If we keep it, a LATER wake's handler could drain + // before that wake's events were processed. + } + bgSessionsAwaitingDrain.removeAll() + } + bgHandlerLock.unlock() + for handler in handlers { DispatchQueue.main.async { handler() } } + } + // MARK: - URLSession delegate public func urlSession(_ session: URLSession, dataTask: URLSessionDataTask, didReceive data: Data) { @@ -356,6 +558,13 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { public func urlSession(_ session: URLSession, task: URLSessionTask, didSendBodyData bytesSent: Int64, totalBytesSent: Int64, totalBytesExpectedToSend: Int64) { + // A chunked part's bytes feed the upload's byte-weighted aggregate. A + // per-task percentage would have no meaning to the consumer. + if let ref = ChunkedCoordinator.partRef(session, task) { + chunked.partProgress(id: ref.id, part: ref.part, incarnation: ref.incarnation, + sent: totalBytesSent) + return + } // 0 rather than -1 when the length is unknown: the documented range is // 0-100, Android reports 0 for the same case, and a negative value renders // as a broken progress bar in a consumer that passes it straight through. @@ -387,6 +596,22 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { for (key, value) in http.allHeaderFields { headers["\(key)"] = "\(value)" } } + // A chunked part's outcome belongs to the coordinator of its upload: + // accept evaluation against the manifest, the window refill, and one + // journaled terminal, only when the whole upload settles. + if let ref = ChunkedCoordinator.partRef(session, task) { + RNBackgroundUpload.lock.lock() + let bodyData = RNBackgroundUpload.responsesData.removeValue(forKey: taskMapKey(session, task)) + RNBackgroundUpload.lock.unlock() + chunked.handlePartCompletion( + id: ref.id, part: ref.part, incarnation: ref.incarnation, + taskKey: taskMapKey(session, task), + statusCode: http != nil ? statusCode : nil, headers: headers, + body: bodyData.flatMap { String(data: $0 as Data, encoding: .utf8) }, + error: error as NSError?) + return + } + RNBackgroundUpload.lock.lock() let bodyData = RNBackgroundUpload.responsesData.removeValue(forKey: taskMapKey(session, task)) RNBackgroundUpload.lastProgressAt[id] = nil @@ -395,8 +620,17 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { // would otherwise linger for the life of the process and a later upload // reusing that id would report a system cancel as a user cancel. let userCancelled = RNBackgroundUpload.userCancelledIds.remove(id) != nil + let removed = RNBackgroundUpload.removedIds.remove(id) != nil RNBackgroundUpload.lock.unlock() + // removeUpload cancelled this task as an explicit release, not as an + // outcome that the consumer awaits. Journal nothing. A non-cancel + // terminal that only raced the removal still reports normally. + if removed, let nsError = error as NSError?, nsError.code == NSURLErrorCancelled { + TaskMap.removeKey(taskMapKey(session, task)) + return + } + let rawBody = bodyData.flatMap { String(data: $0 as Data, encoding: .utf8) } ?? "" let (cappedBody, truncated) = EventJournal.capBody(rawBody) let responseBody = cappedBody ?? "" @@ -412,9 +646,11 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { } if error == nil { - // "completed" only for 2xx or a per-request acceptStatus code; any other - // HTTP response is a terminal http error carrying the full response. - let accepted = (200..<300).contains(statusCode) || acceptStatus(session, task).contains(statusCode) + // "completed" only for a 2xx or a matching per-request accept rule. + // Any other HTTP response is a terminal http error that carries the + // full response. + let accepted = UploadOutcome.isAccepted( + statusCode, body: rawBody, accept: acceptRules(session, task)) if accepted { event.type = "completed" } else { @@ -434,22 +670,24 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { } } - // Journal BEFORE emitting; the emit is best-effort (JS may be dead). - EventJournal.append(event) TaskMap.removeKey(taskMapKey(session, task)) - - let body = event.bridged - let delegate = RNBackgroundUpload.currentDelegate - switch event.type { - case "completed": delegate?.emitCompleted(body) - case "cancelled": delegate?.emitCancelled(body) - default: delegate?.emitError(body) - } + // Journals BEFORE it emits. The emit is best-effort, because JS can be + // dead. + RNBackgroundUpload.journalAndEmit(event) } public func urlSessionDidFinishEvents(forBackgroundURLSession session: URLSession) { guard let identifier = session.configuration.identifier else { return } RNBackgroundUpload.bgHandlerLock.lock() + guard RNBackgroundUpload.bgHandlerDeferrals <= 0 else { + // A relaunch reconcile is in flight. If we hand the system the handler + // now, it can suspend the app before the refill enqueues new part + // tasks. The id is parked. releaseBackgroundCompletionHandlers drains + // it. + RNBackgroundUpload.bgSessionsAwaitingDrain.insert(identifier) + RNBackgroundUpload.bgHandlerLock.unlock() + return + } let handler = RNBackgroundUpload.bgCompletionHandlers.removeValue(forKey: identifier) RNBackgroundUpload.bgHandlerLock.unlock() if let handler { DispatchQueue.main.async { handler() } } @@ -457,8 +695,9 @@ public class RNBackgroundUpload: NSObject, URLSessionDataDelegate { // Classify a transport error to match Android's errorKind taxonomy: a missing or // unreadable source file -> 'file'; other URL-domain errors -> 'network'; anything - // else -> 'unknown'. - private static func errorKind(for error: NSError) -> String { + // else -> 'unknown'. It is internal because the chunked coordinator also + // classifies with it. + static func errorKind(for error: NSError) -> String { switch (error.domain, error.code) { case (NSURLErrorDomain, NSURLErrorFileDoesNotExist), (NSURLErrorDomain, NSURLErrorCannotOpenFile), diff --git a/ios/RNFileUploader.mm b/ios/RNFileUploader.mm index ab8c759f..3bf3c038 100644 --- a/ios/RNFileUploader.mm +++ b/ios/RNFileUploader.mm @@ -79,6 +79,13 @@ - (void)startUpload:(NSDictionary *)options [RNBackgroundUpload.shared startUpload:options resolve:resolve reject:reject]; } +- (void)startChunkedUpload:(NSDictionary *)options + resolve:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject +{ + [RNBackgroundUpload.shared startChunkedUpload:options resolve:resolve reject:reject]; +} + - (void)cancelUpload:(NSString *)id resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject @@ -86,20 +93,11 @@ - (void)cancelUpload:(NSString *)id [RNBackgroundUpload.shared cancelUpload:id resolve:resolve reject:reject]; } -// Chunked transport lands in the iOS engine change; until then the spec -// contract is met by rejecting. -- (void)startChunkedUpload:(NSDictionary *)options - resolve:(RCTPromiseResolveBlock)resolve - reject:(RCTPromiseRejectBlock)reject -{ - reject(@"E_NOT_IMPLEMENTED", @"Chunked uploads are not implemented in this build", nil); -} - - (void)removeUpload:(NSString *)id resolve:(RCTPromiseResolveBlock)resolve reject:(RCTPromiseRejectBlock)reject { - reject(@"E_NOT_IMPLEMENTED", @"removeUpload is not implemented in this build", nil); + [RNBackgroundUpload.shared removeUpload:id resolve:resolve reject:reject]; } - (void)getUnacknowledgedEvents:(RCTPromiseResolveBlock)resolve diff --git a/ios/TaskMap.swift b/ios/TaskMap.swift index 195b2a0a..2748b39d 100644 --- a/ios/TaskMap.swift +++ b/ios/TaskMap.swift @@ -1,23 +1,72 @@ import Foundation -// Durable ":" -> { id, acceptStatus } mapping. +// Durable ":" -> { id, accept, partIndex } mapping. // // Apple documents `taskDescription` only as an uninterpreted app string with no // guarantee it survives process death, and DTS guidance is to persist task // metadata externally keyed by the (stable) taskIdentifier. taskDescription -// stays the primary id; this map is the durable fallback so a task observed -// after relaunch is never orphaned under an unknown id, and so acceptStatus is -// still known when a task completes after the original startUpload options are gone. +// stays the primary id. This map is the durable fallback. Thus a task +// observed after a relaunch is never orphaned under an unknown id, and the +// accept rules are still known when a task completes after the original +// startUpload options are gone. Chunked part tasks carry `partIndex`. Their +// accept rules live in the manifest, so `accept` is nil for them. // // Synchronous serial-queue access; a single JSON file. enum TaskMap { struct Meta: Codable { let id: String - let acceptStatus: [Int] + // All are optional. Thus entries that older builds persisted still + // decode. + var accept: [UploadOutcome.AcceptRule]? + var partIndex: Int? + // Chunked part tasks only: the manifest incarnation that the task was + // created under. It mirrors the taskDescription encoding (see + // ChunkedEngine). + var incarnation: String? + + init(id: String, accept: [UploadOutcome.AcceptRule]?, partIndex: Int?, + incarnation: String? = nil) { + self.id = id + self.accept = accept + self.partIndex = partIndex + self.incarnation = incarnation + } + + private enum CodingKeys: String, CodingKey { + case id, accept, partIndex, incarnation + // Earlier builds persisted `acceptStatus: [Int]` where this build + // persists `accept` rules. The key is read, and never written. Thus a + // task that an older build enqueued keeps its accept rules when it + // completes under this build. This is the same legacy mapping as + // Android's Upload.normalized(). + case acceptStatus + } + + init(from decoder: Decoder) throws { + let c = try decoder.container(keyedBy: CodingKeys.self) + id = try c.decode(String.self, forKey: .id) + partIndex = try c.decodeIfPresent(Int.self, forKey: .partIndex) + incarnation = try c.decodeIfPresent(String.self, forKey: .incarnation) + accept = try c.decodeIfPresent([UploadOutcome.AcceptRule].self, forKey: .accept) + ?? c.decodeIfPresent([Int].self, forKey: .acceptStatus)? + .map { UploadOutcome.AcceptRule(status: $0, bodyIncludes: nil) } + } + + func encode(to encoder: Encoder) throws { + var c = encoder.container(keyedBy: CodingKeys.self) + try c.encode(id, forKey: .id) + try c.encodeIfPresent(accept, forKey: .accept) + try c.encodeIfPresent(partIndex, forKey: .partIndex) + try c.encodeIfPresent(incarnation, forKey: .incarnation) + } } private static let queue = DispatchQueue(label: "ai.openspace.rnbgupload.taskmap") + static func key(_ session: URLSession, _ task: URLSessionTask) -> String { + "\(session.configuration.identifier ?? ""):\(task.taskIdentifier)" + } + private static var fileURL: URL { let base = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask)[0] try? FileManager.default.createDirectory(at: base, withIntermediateDirectories: true) diff --git a/ios/UploadOutcome.swift b/ios/UploadOutcome.swift new file mode 100644 index 00000000..a0e70f7c --- /dev/null +++ b/ios/UploadOutcome.swift @@ -0,0 +1,40 @@ +import Foundation + +// The pure classification of upload outcomes. It mirrors the Android +// UploadOutcome. This is the highest-consequence logic in the uploader, +// because it decides between success and failure. Thus it lives in a small +// unit with no session state, where it is easy to examine. +enum UploadOutcome { + + /// A non-2xx response to treat as success. `bodyIncludes` narrows the rule + /// by a response-body substring. This is necessary when one status has + /// several meanings, and only the message shows the difference (our + /// backend's 409). It is Codable: it is persisted in the chunked manifest + /// and in the TaskMap metadata. + struct AcceptRule: Codable, Equatable { + let status: Int + var bodyIncludes: String? + } + + // Tells whether an HTTP response counts as a successful completion. A 2xx + // always counts, plus any matching per-request accept rule. Anything else, + // 4xx and 5xx included, is an http error, not a completion. + static func isAccepted(_ code: Int, body: String?, accept: [AcceptRule]) -> Bool { + if (200..<300).contains(code) { return true } + return accept.contains { rule in + rule.status == code + && (rule.bodyIncludes == nil || body?.contains(rule.bodyIncludes!) == true) + } + } + + // The bridge delivers `accept` as an array of dictionaries. A malformed + // rule is dropped, not guessed at. JS validates the shape before it + // crosses. + static func parseAcceptRules(_ raw: Any?) -> [AcceptRule] { + guard let rules = raw as? [[String: Any]] else { return [] } + return rules.compactMap { rule in + guard let status = (rule["status"] as? NSNumber)?.intValue else { return nil } + return AcceptRule(status: status, bodyIncludes: rule["bodyIncludes"] as? String) + } + } +} From 532781af76ffcf0b82e9377a233d9eb2d64e10f7 Mon Sep 17 00:00:00 2001 From: Dylan Murphy Date: Thu, 3 Sep 2026 10:52:37 -0400 Subject: [PATCH 4/4] Global cap, example app, docs, v9.0.0 (#35) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR sets the library-wide transmission cap to 4 on both platforms. Before, both platforms sent one request at a time per host (an Android semaphore of 1; iOS `httpMaximumConnectionsPerHost` of 1). On Android, the cap is hard: each request must pass the semaphore. On iOS, the cap is a per-session, connection-level backstop. The window of 3 already bounds the parts of one chunked upload. The PR also: - Adds a chunked demo and a `removeUpload` button to the example app. - Rewrites the README and the CHANGELOG for v9. - Sets the version to 9.0.0. Full suite at the tip: typecheck and lint clean, jest 37, gradle 84, xcodebuild BUILD SUCCEEDED. 🤖 Generated with [Claude Code](https://claude.com/claude-code) --------- Co-authored-by: Claude Fable 5 --- CHANGELOG.md | 61 ++++++ README.md | 178 +++++++++++++++--- .../backgroundupload/UploadTransport.kt | 9 +- example/RNBGUExample/App.tsx | 89 ++++++++- example/RNBGUExample/tsconfig.json | 5 +- ios/RNBackgroundUpload.swift | 5 +- package.json | 2 +- 7 files changed, 308 insertions(+), 41 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aaf6abc..a22f35b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,64 @@ +## 9.0.0 + +Chunked uploads move into the library: one file, many part requests, one upload +id and event stream. The consumer authors the parts (URL, headers, byte range) +once; the library owns transport, the bytes, and resume. See the README's +"Chunked uploads" section. + +Breaking: +- **Listeners are global-only.** `addListener(event, uploadId, callback)` is + gone; use `addListener(event, callback)` and discriminate on `data.id`. +- **`acceptStatus: number[]` is replaced by `accept` rules** on all uploads: + `accept: [{ status, bodyIncludes? }]`. `bodyIncludes` narrows by + response-body substring, for statuses that carry several meanings. +- **`customUploadId` is renamed to `id`** on all upload types. +- **`android.maxRetries` is removed.** Retry policy belongs to the library; + chunked uploads are bounded by `expiresAt` instead. +- **`ios.getUploadStatus` is removed.** Its only use was pre-dispatch dedupe, + which idempotent `startUpload` (below) makes unnecessary. +- **Per-upload notification text is removed.** Set it once with `configure()`; + it is persisted natively so a worker relaunched by WorkManager with no JS + running shows the same text. Per-upload `android` options reduce to + `noNotification`. + +Added: +- **Chunked uploads**: `startUpload({ type: 'chunked', id, path, + parts, accept?, expiresAt, wifiOnly? })`. The library takes ownership of the + file at `startUpload` and deletes it only after a `completed` event is + acknowledged; every other terminal outcome keeps the manifest and bytes for + resume or recreate. +- **`startUpload` is idempotent for every upload, always.** Re-calling with a + running id is never an error; for chunked uploads it reconciles — accepted + parts are skipped, the rest continue with the new call's headers (how a fresh + auth token reaches parts stalled on 401). On iOS the guarantee is race-free: + concurrent same-id calls are serialized natively, so they can never enqueue a + duplicate task. +- **Recreate under the same id**: calling `startUpload` with an existing id and + a *different* parts array replaces the parts over the owned bytes — every + part resets to unsent, and the new ranges must tile the same total size. + Accepted only while the upload is not running (stalled on a terminal error, + expired, or cancelled); rejected while it runs. This is how a consumer + re-uploads under a fresh server uploadId after the old one dies. +- **`expiresAt` / `errorKind: 'expired'`**: a chunked upload past its required + deadline journals a terminal `error` and stops, keeping the bytes. Within the + deadline, transient failures retry on backoff with no attempt cap. +- **`removeUpload(uploadId)`** releases a kept manifest and bytes. +- **`configure(options)`** one-time setup (Android notification text/identity). +- **`chunkPlan(sizeBytes, { min?, max? })`**: the deterministic range splitter, + exposed so the part count told to the server and the parts array derive from + one result. +- Concurrency: on Android a hard cap of 4 concurrent requests across all + uploads — every request passes the shared transfer semaphore (previously + fully serial), with a chunked upload's parts windowed inside it. On iOS a + per-session connection-level backstop: `httpMaximumConnectionsPerHost = 4` + (previously 1), with chunked parts bounded by the per-upload window of 3. + +Fixed: +- Android jobs enqueued by a v8 build replay safely after upgrading. WorkManager + can hand a v9 worker a job serialized by v8 (`acceptStatus`, no `accept`); + the worker now normalizes the legacy shape instead of crashing after the file + has fully transmitted and re-sending it on every retry. + ## 8.1.0 Added `android.noNotification`, which uploads a file without posting a progress diff --git a/README.md b/README.md index 1b258fbc..2947d31e 100644 --- a/README.md +++ b/README.md @@ -54,31 +54,102 @@ generated header is Objective-C++ only — so the handler lives on `RNBackground ```js import Upload from 'react-native-background-upload'; -const options = { - url: 'https://myservice.com/path/to/post', - path: 'file://path/to/file/on/device', - method: 'POST', - type: 'raw', - headers: { 'content-type': 'application/octet-stream' }, - // Optional. Treat these non-2xx statuses as success (e.g. an idempotent - // create that conflicts). Any other non-2xx is an 'error' with errorKind 'http'. - acceptStatus: [409], -}; - // Optional. Call one time at app startup to set the Android notification text. // The library keeps the text in native storage. Thus a worker relaunched with // no JS shows the same text. If you do not call configure(), the library uses // default text and makes its own channel. The call does nothing on iOS. Upload.configure({ android: { notificationTitle: 'Uploading…' } }); -const uploadId = await Upload.startUpload(options); - +// Listeners are global. Every event carries the upload's id. Upload.addListener('progress', ({ id, progress }) => {}); +// responseCode/responseBody are set for simple uploads only. A chunked +// 'completed' carries neither, because no single response represents N parts. Upload.addListener('completed', ({ id, responseCode, responseBody }) => {}); Upload.addListener('error', ({ id, error, errorKind, responseCode }) => {}); Upload.addListener('cancelled', ({ id, cancelReason }) => {}); + +const uploadId = await Upload.startUpload({ + type: 'raw', + url: 'https://myservice.com/path/to/post', + path: 'file://path/to/file/on/device', + method: 'POST', + headers: { 'content-type': 'application/octet-stream' }, + // Optional. Non-2xx responses to treat as success (for example, an + // idempotent create that conflicts). Each other non-2xx response is an + // 'error' with errorKind 'http'. + accept: [{ status: 409, bodyIncludes: 'already completed' }], +}); +``` + +## Chunked uploads + +A `type: 'chunked'` upload sends one file as many part requests but stays one +logical upload: one id, one event stream, byte-weighted `progress`, and +`completed` only when every part has been accepted. You author the parts — URL, +headers, byte range — once, at creation; the library owns the transport and +never constructs or edits a protocol field. Author the ranges with `chunkPlan` +so the part count you tell your server and the parts the library sends derive +from the same array: + +```js +const size = (await stat(path)).size; +const ranges = Upload.chunkPlan(size, { min: 8 * 2 ** 20, max: 20 * 2 ** 20 }); +// Tell your server ranges.length parts, then: +await Upload.startUpload({ + type: 'chunked', + id: myDurableId, // required + path, // see file ownership below + parts: ranges.map((range, i) => ({ + url: partUrl(i + 1), + headers: { + Authorization: token, + 'Content-Type': 'application/octet-stream', + 'Content-Range': `bytes ${range.start}-${range.end - 1}/${size}`, + }, + range, // bytes, end exclusive + })), + accept: [{ status: 409, bodyIncludes: 'already completed' }], + expiresAt: Date.now() + 14 * 24 * 60 * 60 * 1000, // required, epoch ms +}); ``` +**File ownership.** The library takes the file: an O(1) rename into its own +directory at `startUpload`. Nothing your app does afterward (cache sweeps, +logout cleanup) can destroy the bytes mid-upload. The file is deleted in +exactly one case — a `completed` event has been acknowledged via `ackEvents`. +Copy the file first if you need it afterward. + +**Resume is re-calling `startUpload`.** The parts are persisted in a native +manifest, so crash recovery, resume after `cancelUpload`, resume after expiry, +and refreshing auth headers are all the same call: `startUpload` again with the +same id and the same part ranges/URLs. Parts already accepted are skipped; the +rest continue with the new call's headers and `expiresAt` (this is how a fresh +token reaches parts that stalled on 401). Once a manifest exists, `path` is +ignored — the library's owned bytes are the source of truth. + +**Recreate is the same call with different parts.** When the old server upload +is dead (for example, swept server-side), author fresh part URLs and call +`startUpload` with the same id and the new parts array. The owned bytes are +kept, the parts are replaced, and every part resets to unsent; the new ranges +must tile the same total size. A recreate is accepted only while the upload is +not running — stalled on a terminal error, expired, or cancelled. While it is +running, a differing parts array is rejected: that is a consumer bug, not a +recreate. + +**Lifetime.** Within `expiresAt`, transient failures (network, 5xx) retry on +exponential backoff with no attempt cap. Past it, the library journals an +`error` with `errorKind: 'expired'` and stops — keeping the manifest and bytes, +so you can resume the same server upload with a later `expiresAt`, or recreate +under a new one. When neither is wanted, release them with `removeUpload`. + +Choosing a value: `expiresAt` is when your app *hears about* a stuck upload, +not when data is lost — bytes survive expiry. Pick something well inside your +backend's own cleanup horizon so expiry fires while the server upload is still +resumable, and generous enough for real offline stretches. The OpenSpace +backend prunes incomplete multipart uploads 31 days after creation +(`UploadPartCleanup`); Diana passes 14 days, leaving a 17-day window where an +expired upload can still resume the same server uploadId. + # Reliable delivery Terminal events (`completed` / `error` / `cancelled`) are journaled natively @@ -100,11 +171,13 @@ const live = await Upload.getAllUploads(); // [{ id, state, ... }] ``` Notes: -- **`completed` fires only for 2xx** (or a request's `acceptStatus`). Every other - HTTP response is an `error` with `errorKind: 'http'` and the response attached — - a 400 is an error, not a completion. -- `errorKind` is `'http' | 'network' | 'file' | 'unknown'`. Retry transport - failures; treat client errors as terminal. +- **`completed` fires only for 2xx** (or a response matching the request's + `accept` rules). Every other HTTP response is an `error` with + `errorKind: 'http'` and the response attached — a 400 is an error, not a + completion. +- `errorKind` is `'http' | 'network' | 'file' | 'expired' | 'unknown'`. Retry + transport failures; treat client errors as terminal; `expired` means a chunked + upload's `expiresAt` passed (see Chunked uploads for recovery). - `cancelReason` distinguishes a user cancel (`'user'`) from a system kill (`'system'`). - Duplicate journal entries for one upload id are possible if the process dies at @@ -126,24 +199,52 @@ call replaces the whole config). A no-op on iOS, which has no library notification. ### `startUpload(options): Promise` -Starts an upload; resolves to its id. Rejects only on a bad option (missing/invalid -`url` or `path`) — transport failures and HTTP error responses arrive later as -`error` events, not a rejection. Idempotent for a given `id`: calling -it again while that upload is pending or running resolves with the same id instead -of starting a duplicate. +Starts an upload; resolves to its id. Discriminated on `options.type`: `'raw'` +sends the whole file as one request body, `'chunked'` sends the authored parts +(see Chunked uploads). Rejects (or, for malformed chunked input, throws +synchronously) only on bad options — transport failures and HTTP error responses +arrive later as `error` events, not a rejection. + +**Idempotent for every upload, always.** Calling `startUpload` again with an id +that is already pending or running is never an error: a raw upload resolves with +the same id instead of starting a duplicate; a chunked upload reconciles — parts +already accepted are skipped, the rest continue with the new call's headers. No +pre-dispatch dedupe is needed on your side. + +Options for `type: 'raw'`: | Option | Type | Notes | | --- | --- | --- | | `url` | string | Required. | | `path` | string | Required. Local file path (`file://…`). URIs are not escaped for you. | -| `type` | `'raw'` | Only `raw` is supported. | | `method` | string | Default `POST`. | | `headers` | object | HTTP headers. | | `id` | string | Defaults to a generated UUID. | | `wifiOnly` | boolean | Wait for wifi before/while uploading. | -| `acceptStatus` | number[] | Non-2xx statuses to treat as success. | +| `accept` | AcceptRule[] | Non-2xx responses to treat as success — see Accept rules. | | `android` | object | Optional. `noNotification` (default false) — see Silent uploads. Notification text is set once via `configure()`, not per upload. | +Options for `type: 'chunked'`: + +| Option | Type | Notes | +| --- | --- | --- | +| `id` | string | Required — your durable id. | +| `path` | string | Required. The library takes ownership of the file — see Chunked uploads. | +| `parts` | array | Required. `{ url, headers, range: { start, end } }` per part; ranges in bytes, end exclusive. Sent verbatim as PUTs. | +| `expiresAt` | number | Required, epoch ms. Past it: terminal `error` with `errorKind: 'expired'`. | +| `accept` | AcceptRule[] | See Accept rules. | +| `wifiOnly` | boolean | Wait for wifi before/while uploading. | +| `android` | object | Same as raw. | + +#### Accept rules + +`accept: Array<{ status: number, bodyIncludes?: string }>` — non-2xx responses +to treat as success, for both upload types. `bodyIncludes` narrows a rule by +response-body substring, for servers where one status carries several meanings +distinguishable only by message. A response matching a rule completes the +request (for chunked, marks the part accepted); any other non-2xx is an `error` +with `errorKind: 'http'`. + #### Silent uploads (Android) `android: { noNotification: true }` uploads a file without posting a progress @@ -160,11 +261,28 @@ All uploads share one notification (identified by the configured ones included. ### `cancelUpload(uploadId): Promise` -Cancels an upload. Fires a `cancelled` event with `cancelReason: 'user'`. +Cancels an upload. Fires a `cancelled` event with `cancelReason: 'user'`. For a +chunked upload this cancels in-flight requests but keeps the manifest and bytes — +the next `startUpload` with the same id resumes it (there is no separate pause +API). + +### `removeUpload(uploadId): Promise` +Releases an upload's native manifest and bytes. Every terminal outcome other +than an acked `completed` (expired, error, cancelled) keeps both so you can +resume or recreate; call this once neither is wanted. + +### `chunkPlan(sizeBytes, { min?, max? }): Array<{ start, end }>` +Splits a byte count into contiguous, end-exclusive ranges: a deterministic +greedy walk of `max`-sized chunks (default 20MB), with a final remainder smaller +than `min` (default 8MB) absorbed into the previous chunk. A file smaller than +`min` is a single chunk. Pure and deterministic on purpose: call it once and +derive both your server's part count and the `parts` array from the same result, +so the two can never disagree. ### `addListener(eventType, listener): EventSubscription` -Listen for `'progress' | 'error' | 'completed' | 'cancelled'` across all uploads; -every event carries the upload's `id`. Call `.remove()` on the result to +`addListener(event: 'progress' | 'error' | 'completed' | 'cancelled', callback)`. +Listeners are global — there is no per-upload subscription; every event carries +the upload's `id`, so discriminate on it. Call `.remove()` on the result to unsubscribe. ### `getUnacknowledgedEvents(): Promise` @@ -184,8 +302,8 @@ Fires when the Android progress notification is pressed. No event data. | Event | Data | | --- | --- | | `progress` | `{ id, progress: 0-100 }` | -| `completed` | `{ id, responseCode, responseBody, responseHeaders?, eventId? }` | -| `error` | `{ id, error, errorKind?, responseCode?, responseBody?, responseHeaders? }` | +| `completed` | `{ id, responseCode?, responseBody?, responseHeaders?, eventId? }` — response fields on simple uploads only; a chunked `completed` carries none (no single response represents N parts) | +| `error` | `{ id, error, errorKind?, partIndex?, responseCode?, responseBody?, responseHeaders? }` | | `cancelled` | `{ id, cancelReason?: 'user' | 'system' }` | # Contributing diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt index 06718b01..cc8093ea 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt @@ -16,10 +16,11 @@ private const val REQUEST_TIMEOUT = 24L private val REQUEST_TIMEOUT_UNIT = TimeUnit.HOURS // The number of requests transmitting at one time across ALL uploads, chunked -// parts included. A semaphore controls this, not OkHttp's connection limits, -// because those limits add a delay between requests. The design's library-wide -// cap is 4. The change from 1 to 4 lands with the hardening slice, not here. -internal const val MAX_TRANSFER_CONCURRENCY = 1 +// parts included. This is the design's library-wide cap of 4. Every request, +// a simple upload or a chunked part, must pass this semaphore. Thus on +// Android the cap is hard. A semaphore controls this, not OkHttp's connection +// limits, because those limits add a delay between requests. +internal const val MAX_TRANSFER_CONCURRENCY = 4 internal val transferSemaphore = Semaphore(MAX_TRANSFER_CONCURRENCY) // Use Okhttp as it provides the most standard behaviors even though it's not coroutine friendly diff --git a/example/RNBGUExample/App.tsx b/example/RNBGUExample/App.tsx index f656b38f..f7dbc9f7 100644 --- a/example/RNBGUExample/App.tsx +++ b/example/RNBGUExample/App.tsx @@ -17,9 +17,11 @@ import { Button, } from 'react-native'; import notifee, {AndroidImportance} from '@notifee/react-native'; -import {Colors} from 'react-native/Libraries/NewAppScreen'; -import Upload, {UploadOptions} from 'react-native-background-upload'; +import Upload, { + ChunkedUploadOptions, + UploadOptions, +} from 'react-native-background-upload'; import * as RNFS from 'react-native-fs'; @@ -27,6 +29,7 @@ const TEST_FILE = `${RNFS.DocumentDirectoryPath}/1MB.bin`; const TEST_FILE_URL = 'https://gist.githubusercontent.com/khaykov/a6105154becce4c0530da38e723c2330/raw/41ab415ac41c93a198f7da5b47d604956157c5c3/gistfile1.txt'; const UPLOAD_URL = 'https://httpbin.org/post'; +const CHUNKED_UPLOAD_URL = 'https://httpbin.org/put'; const NOTIFICATION_CHANNEL = 'RNBGUExample'; const App = () => { @@ -78,7 +81,7 @@ const App = () => { .then(() => setTestFileDownload('downloaded')); }, []); - const onPressUpload = async () => { + const ensureNotificationChannel = async () => { await notifee.requestPermission({alert: true, sound: true}); await notifee.createChannel({ @@ -86,6 +89,10 @@ const App = () => { name: NOTIFICATION_CHANNEL, importance: AndroidImportance.LOW, }); + }; + + const onPressUpload = async () => { + await ensureNotificationChannel(); const uploadOpts: UploadOptions = { type: 'raw', @@ -110,6 +117,53 @@ const App = () => { }); }; + const onPressChunkedUpload = async () => { + await ensureNotificationChannel(); + + // The library takes ownership of a chunked upload's file. It renames the + // file into its own directory. Thus we upload a copy, and the test file + // stays available. + const chunkedFile = `${RNFS.DocumentDirectoryPath}/chunked.bin`; + if (await RNFS.exists('file://' + chunkedFile)) { + await RNFS.unlink(chunkedFile); + } + await RNFS.copyFile(TEST_FILE, chunkedFile); + + // A small min and max, so the 1MB test file still splits into some parts. + // Production callers use the server's real part-size limits. + const {size} = await RNFS.stat(chunkedFile); + const ranges = Upload.chunkPlan(size, {min: 128 * 1024, max: 256 * 1024}); + + const uploadOpts: ChunkedUploadOptions = { + type: 'chunked', + id: 'chunked-demo', + path: chunkedFile, + parts: ranges.map((range, i) => ({ + url: `${CHUNKED_UPLOAD_URL}?partNum=${i + 1}`, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Range': `bytes ${range.start}-${range.end - 1}/${size}`, + }, + range, + })), + expiresAt: Date.now() + 24 * 60 * 60 * 1000, + }; + + Upload.startUpload(uploadOpts) + .then(uploadId => { + console.log( + `Chunked upload started: ${uploadId} (${ranges.length} parts)`, + ); + setUploadId(uploadId); + setProgress(0); + }) + .catch(function (err) { + setUploadId(undefined); + setProgress(undefined); + console.log('Chunked upload error!', err); + }); + }; + return ( <> @@ -126,6 +180,7 @@ const App = () => {