diff --git a/CHANGELOG.md b/CHANGELOG.md index 2aaf6abc..a22f35b0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,64 @@ +## 9.0.0 + +Chunked uploads move into the library: one file, many part requests, one upload +id and event stream. The consumer authors the parts (URL, headers, byte range) +once; the library owns transport, the bytes, and resume. See the README's +"Chunked uploads" section. + +Breaking: +- **Listeners are global-only.** `addListener(event, uploadId, callback)` is + gone; use `addListener(event, callback)` and discriminate on `data.id`. +- **`acceptStatus: number[]` is replaced by `accept` rules** on all uploads: + `accept: [{ status, bodyIncludes? }]`. `bodyIncludes` narrows by + response-body substring, for statuses that carry several meanings. +- **`customUploadId` is renamed to `id`** on all upload types. +- **`android.maxRetries` is removed.** Retry policy belongs to the library; + chunked uploads are bounded by `expiresAt` instead. +- **`ios.getUploadStatus` is removed.** Its only use was pre-dispatch dedupe, + which idempotent `startUpload` (below) makes unnecessary. +- **Per-upload notification text is removed.** Set it once with `configure()`; + it is persisted natively so a worker relaunched by WorkManager with no JS + running shows the same text. Per-upload `android` options reduce to + `noNotification`. + +Added: +- **Chunked uploads**: `startUpload({ type: 'chunked', id, path, + parts, accept?, expiresAt, wifiOnly? })`. The library takes ownership of the + file at `startUpload` and deletes it only after a `completed` event is + acknowledged; every other terminal outcome keeps the manifest and bytes for + resume or recreate. +- **`startUpload` is idempotent for every upload, always.** Re-calling with a + running id is never an error; for chunked uploads it reconciles — accepted + parts are skipped, the rest continue with the new call's headers (how a fresh + auth token reaches parts stalled on 401). On iOS the guarantee is race-free: + concurrent same-id calls are serialized natively, so they can never enqueue a + duplicate task. +- **Recreate under the same id**: calling `startUpload` with an existing id and + a *different* parts array replaces the parts over the owned bytes — every + part resets to unsent, and the new ranges must tile the same total size. + Accepted only while the upload is not running (stalled on a terminal error, + expired, or cancelled); rejected while it runs. This is how a consumer + re-uploads under a fresh server uploadId after the old one dies. +- **`expiresAt` / `errorKind: 'expired'`**: a chunked upload past its required + deadline journals a terminal `error` and stops, keeping the bytes. Within the + deadline, transient failures retry on backoff with no attempt cap. +- **`removeUpload(uploadId)`** releases a kept manifest and bytes. +- **`configure(options)`** one-time setup (Android notification text/identity). +- **`chunkPlan(sizeBytes, { min?, max? })`**: the deterministic range splitter, + exposed so the part count told to the server and the parts array derive from + one result. +- Concurrency: on Android a hard cap of 4 concurrent requests across all + uploads — every request passes the shared transfer semaphore (previously + fully serial), with a chunked upload's parts windowed inside it. On iOS a + per-session connection-level backstop: `httpMaximumConnectionsPerHost = 4` + (previously 1), with chunked parts bounded by the per-upload window of 3. + +Fixed: +- Android jobs enqueued by a v8 build replay safely after upgrading. WorkManager + can hand a v9 worker a job serialized by v8 (`acceptStatus`, no `accept`); + the worker now normalizes the legacy shape instead of crashing after the file + has fully transmitted and re-sending it on every retry. + ## 8.1.0 Added `android.noNotification`, which uploads a file without posting a progress diff --git a/README.md b/README.md index 1b258fbc..2947d31e 100644 --- a/README.md +++ b/README.md @@ -54,31 +54,102 @@ generated header is Objective-C++ only — so the handler lives on `RNBackground ```js import Upload from 'react-native-background-upload'; -const options = { - url: 'https://myservice.com/path/to/post', - path: 'file://path/to/file/on/device', - method: 'POST', - type: 'raw', - headers: { 'content-type': 'application/octet-stream' }, - // Optional. Treat these non-2xx statuses as success (e.g. an idempotent - // create that conflicts). Any other non-2xx is an 'error' with errorKind 'http'. - acceptStatus: [409], -}; - // Optional. Call one time at app startup to set the Android notification text. // The library keeps the text in native storage. Thus a worker relaunched with // no JS shows the same text. If you do not call configure(), the library uses // default text and makes its own channel. The call does nothing on iOS. Upload.configure({ android: { notificationTitle: 'Uploading…' } }); -const uploadId = await Upload.startUpload(options); - +// Listeners are global. Every event carries the upload's id. Upload.addListener('progress', ({ id, progress }) => {}); +// responseCode/responseBody are set for simple uploads only. A chunked +// 'completed' carries neither, because no single response represents N parts. Upload.addListener('completed', ({ id, responseCode, responseBody }) => {}); Upload.addListener('error', ({ id, error, errorKind, responseCode }) => {}); Upload.addListener('cancelled', ({ id, cancelReason }) => {}); + +const uploadId = await Upload.startUpload({ + type: 'raw', + url: 'https://myservice.com/path/to/post', + path: 'file://path/to/file/on/device', + method: 'POST', + headers: { 'content-type': 'application/octet-stream' }, + // Optional. Non-2xx responses to treat as success (for example, an + // idempotent create that conflicts). Each other non-2xx response is an + // 'error' with errorKind 'http'. + accept: [{ status: 409, bodyIncludes: 'already completed' }], +}); +``` + +## Chunked uploads + +A `type: 'chunked'` upload sends one file as many part requests but stays one +logical upload: one id, one event stream, byte-weighted `progress`, and +`completed` only when every part has been accepted. You author the parts — URL, +headers, byte range — once, at creation; the library owns the transport and +never constructs or edits a protocol field. Author the ranges with `chunkPlan` +so the part count you tell your server and the parts the library sends derive +from the same array: + +```js +const size = (await stat(path)).size; +const ranges = Upload.chunkPlan(size, { min: 8 * 2 ** 20, max: 20 * 2 ** 20 }); +// Tell your server ranges.length parts, then: +await Upload.startUpload({ + type: 'chunked', + id: myDurableId, // required + path, // see file ownership below + parts: ranges.map((range, i) => ({ + url: partUrl(i + 1), + headers: { + Authorization: token, + 'Content-Type': 'application/octet-stream', + 'Content-Range': `bytes ${range.start}-${range.end - 1}/${size}`, + }, + range, // bytes, end exclusive + })), + accept: [{ status: 409, bodyIncludes: 'already completed' }], + expiresAt: Date.now() + 14 * 24 * 60 * 60 * 1000, // required, epoch ms +}); ``` +**File ownership.** The library takes the file: an O(1) rename into its own +directory at `startUpload`. Nothing your app does afterward (cache sweeps, +logout cleanup) can destroy the bytes mid-upload. The file is deleted in +exactly one case — a `completed` event has been acknowledged via `ackEvents`. +Copy the file first if you need it afterward. + +**Resume is re-calling `startUpload`.** The parts are persisted in a native +manifest, so crash recovery, resume after `cancelUpload`, resume after expiry, +and refreshing auth headers are all the same call: `startUpload` again with the +same id and the same part ranges/URLs. Parts already accepted are skipped; the +rest continue with the new call's headers and `expiresAt` (this is how a fresh +token reaches parts that stalled on 401). Once a manifest exists, `path` is +ignored — the library's owned bytes are the source of truth. + +**Recreate is the same call with different parts.** When the old server upload +is dead (for example, swept server-side), author fresh part URLs and call +`startUpload` with the same id and the new parts array. The owned bytes are +kept, the parts are replaced, and every part resets to unsent; the new ranges +must tile the same total size. A recreate is accepted only while the upload is +not running — stalled on a terminal error, expired, or cancelled. While it is +running, a differing parts array is rejected: that is a consumer bug, not a +recreate. + +**Lifetime.** Within `expiresAt`, transient failures (network, 5xx) retry on +exponential backoff with no attempt cap. Past it, the library journals an +`error` with `errorKind: 'expired'` and stops — keeping the manifest and bytes, +so you can resume the same server upload with a later `expiresAt`, or recreate +under a new one. When neither is wanted, release them with `removeUpload`. + +Choosing a value: `expiresAt` is when your app *hears about* a stuck upload, +not when data is lost — bytes survive expiry. Pick something well inside your +backend's own cleanup horizon so expiry fires while the server upload is still +resumable, and generous enough for real offline stretches. The OpenSpace +backend prunes incomplete multipart uploads 31 days after creation +(`UploadPartCleanup`); Diana passes 14 days, leaving a 17-day window where an +expired upload can still resume the same server uploadId. + # Reliable delivery Terminal events (`completed` / `error` / `cancelled`) are journaled natively @@ -100,11 +171,13 @@ const live = await Upload.getAllUploads(); // [{ id, state, ... }] ``` Notes: -- **`completed` fires only for 2xx** (or a request's `acceptStatus`). Every other - HTTP response is an `error` with `errorKind: 'http'` and the response attached — - a 400 is an error, not a completion. -- `errorKind` is `'http' | 'network' | 'file' | 'unknown'`. Retry transport - failures; treat client errors as terminal. +- **`completed` fires only for 2xx** (or a response matching the request's + `accept` rules). Every other HTTP response is an `error` with + `errorKind: 'http'` and the response attached — a 400 is an error, not a + completion. +- `errorKind` is `'http' | 'network' | 'file' | 'expired' | 'unknown'`. Retry + transport failures; treat client errors as terminal; `expired` means a chunked + upload's `expiresAt` passed (see Chunked uploads for recovery). - `cancelReason` distinguishes a user cancel (`'user'`) from a system kill (`'system'`). - Duplicate journal entries for one upload id are possible if the process dies at @@ -126,24 +199,52 @@ call replaces the whole config). A no-op on iOS, which has no library notification. ### `startUpload(options): Promise` -Starts an upload; resolves to its id. Rejects only on a bad option (missing/invalid -`url` or `path`) — transport failures and HTTP error responses arrive later as -`error` events, not a rejection. Idempotent for a given `id`: calling -it again while that upload is pending or running resolves with the same id instead -of starting a duplicate. +Starts an upload; resolves to its id. Discriminated on `options.type`: `'raw'` +sends the whole file as one request body, `'chunked'` sends the authored parts +(see Chunked uploads). Rejects (or, for malformed chunked input, throws +synchronously) only on bad options — transport failures and HTTP error responses +arrive later as `error` events, not a rejection. + +**Idempotent for every upload, always.** Calling `startUpload` again with an id +that is already pending or running is never an error: a raw upload resolves with +the same id instead of starting a duplicate; a chunked upload reconciles — parts +already accepted are skipped, the rest continue with the new call's headers. No +pre-dispatch dedupe is needed on your side. + +Options for `type: 'raw'`: | Option | Type | Notes | | --- | --- | --- | | `url` | string | Required. | | `path` | string | Required. Local file path (`file://…`). URIs are not escaped for you. | -| `type` | `'raw'` | Only `raw` is supported. | | `method` | string | Default `POST`. | | `headers` | object | HTTP headers. | | `id` | string | Defaults to a generated UUID. | | `wifiOnly` | boolean | Wait for wifi before/while uploading. | -| `acceptStatus` | number[] | Non-2xx statuses to treat as success. | +| `accept` | AcceptRule[] | Non-2xx responses to treat as success — see Accept rules. | | `android` | object | Optional. `noNotification` (default false) — see Silent uploads. Notification text is set once via `configure()`, not per upload. | +Options for `type: 'chunked'`: + +| Option | Type | Notes | +| --- | --- | --- | +| `id` | string | Required — your durable id. | +| `path` | string | Required. The library takes ownership of the file — see Chunked uploads. | +| `parts` | array | Required. `{ url, headers, range: { start, end } }` per part; ranges in bytes, end exclusive. Sent verbatim as PUTs. | +| `expiresAt` | number | Required, epoch ms. Past it: terminal `error` with `errorKind: 'expired'`. | +| `accept` | AcceptRule[] | See Accept rules. | +| `wifiOnly` | boolean | Wait for wifi before/while uploading. | +| `android` | object | Same as raw. | + +#### Accept rules + +`accept: Array<{ status: number, bodyIncludes?: string }>` — non-2xx responses +to treat as success, for both upload types. `bodyIncludes` narrows a rule by +response-body substring, for servers where one status carries several meanings +distinguishable only by message. A response matching a rule completes the +request (for chunked, marks the part accepted); any other non-2xx is an `error` +with `errorKind: 'http'`. + #### Silent uploads (Android) `android: { noNotification: true }` uploads a file without posting a progress @@ -160,11 +261,28 @@ All uploads share one notification (identified by the configured ones included. ### `cancelUpload(uploadId): Promise` -Cancels an upload. Fires a `cancelled` event with `cancelReason: 'user'`. +Cancels an upload. Fires a `cancelled` event with `cancelReason: 'user'`. For a +chunked upload this cancels in-flight requests but keeps the manifest and bytes — +the next `startUpload` with the same id resumes it (there is no separate pause +API). + +### `removeUpload(uploadId): Promise` +Releases an upload's native manifest and bytes. Every terminal outcome other +than an acked `completed` (expired, error, cancelled) keeps both so you can +resume or recreate; call this once neither is wanted. + +### `chunkPlan(sizeBytes, { min?, max? }): Array<{ start, end }>` +Splits a byte count into contiguous, end-exclusive ranges: a deterministic +greedy walk of `max`-sized chunks (default 20MB), with a final remainder smaller +than `min` (default 8MB) absorbed into the previous chunk. A file smaller than +`min` is a single chunk. Pure and deterministic on purpose: call it once and +derive both your server's part count and the `parts` array from the same result, +so the two can never disagree. ### `addListener(eventType, listener): EventSubscription` -Listen for `'progress' | 'error' | 'completed' | 'cancelled'` across all uploads; -every event carries the upload's `id`. Call `.remove()` on the result to +`addListener(event: 'progress' | 'error' | 'completed' | 'cancelled', callback)`. +Listeners are global — there is no per-upload subscription; every event carries +the upload's `id`, so discriminate on it. Call `.remove()` on the result to unsubscribe. ### `getUnacknowledgedEvents(): Promise` @@ -184,8 +302,8 @@ Fires when the Android progress notification is pressed. No event data. | Event | Data | | --- | --- | | `progress` | `{ id, progress: 0-100 }` | -| `completed` | `{ id, responseCode, responseBody, responseHeaders?, eventId? }` | -| `error` | `{ id, error, errorKind?, responseCode?, responseBody?, responseHeaders? }` | +| `completed` | `{ id, responseCode?, responseBody?, responseHeaders?, eventId? }` — response fields on simple uploads only; a chunked `completed` carries none (no single response represents N parts) | +| `error` | `{ id, error, errorKind?, partIndex?, responseCode?, responseBody?, responseHeaders? }` | | `cancelled` | `{ id, cancelReason?: 'user' | 'system' }` | # Contributing diff --git a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt index 06718b01..cc8093ea 100644 --- a/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt +++ b/android/src/main/java/ai/openspace/backgroundupload/UploadTransport.kt @@ -16,10 +16,11 @@ private const val REQUEST_TIMEOUT = 24L private val REQUEST_TIMEOUT_UNIT = TimeUnit.HOURS // The number of requests transmitting at one time across ALL uploads, chunked -// parts included. A semaphore controls this, not OkHttp's connection limits, -// because those limits add a delay between requests. The design's library-wide -// cap is 4. The change from 1 to 4 lands with the hardening slice, not here. -internal const val MAX_TRANSFER_CONCURRENCY = 1 +// parts included. This is the design's library-wide cap of 4. Every request, +// a simple upload or a chunked part, must pass this semaphore. Thus on +// Android the cap is hard. A semaphore controls this, not OkHttp's connection +// limits, because those limits add a delay between requests. +internal const val MAX_TRANSFER_CONCURRENCY = 4 internal val transferSemaphore = Semaphore(MAX_TRANSFER_CONCURRENCY) // Use Okhttp as it provides the most standard behaviors even though it's not coroutine friendly diff --git a/example/RNBGUExample/App.tsx b/example/RNBGUExample/App.tsx index f656b38f..f7dbc9f7 100644 --- a/example/RNBGUExample/App.tsx +++ b/example/RNBGUExample/App.tsx @@ -17,9 +17,11 @@ import { Button, } from 'react-native'; import notifee, {AndroidImportance} from '@notifee/react-native'; -import {Colors} from 'react-native/Libraries/NewAppScreen'; -import Upload, {UploadOptions} from 'react-native-background-upload'; +import Upload, { + ChunkedUploadOptions, + UploadOptions, +} from 'react-native-background-upload'; import * as RNFS from 'react-native-fs'; @@ -27,6 +29,7 @@ const TEST_FILE = `${RNFS.DocumentDirectoryPath}/1MB.bin`; const TEST_FILE_URL = 'https://gist.githubusercontent.com/khaykov/a6105154becce4c0530da38e723c2330/raw/41ab415ac41c93a198f7da5b47d604956157c5c3/gistfile1.txt'; const UPLOAD_URL = 'https://httpbin.org/post'; +const CHUNKED_UPLOAD_URL = 'https://httpbin.org/put'; const NOTIFICATION_CHANNEL = 'RNBGUExample'; const App = () => { @@ -78,7 +81,7 @@ const App = () => { .then(() => setTestFileDownload('downloaded')); }, []); - const onPressUpload = async () => { + const ensureNotificationChannel = async () => { await notifee.requestPermission({alert: true, sound: true}); await notifee.createChannel({ @@ -86,6 +89,10 @@ const App = () => { name: NOTIFICATION_CHANNEL, importance: AndroidImportance.LOW, }); + }; + + const onPressUpload = async () => { + await ensureNotificationChannel(); const uploadOpts: UploadOptions = { type: 'raw', @@ -110,6 +117,53 @@ const App = () => { }); }; + const onPressChunkedUpload = async () => { + await ensureNotificationChannel(); + + // The library takes ownership of a chunked upload's file. It renames the + // file into its own directory. Thus we upload a copy, and the test file + // stays available. + const chunkedFile = `${RNFS.DocumentDirectoryPath}/chunked.bin`; + if (await RNFS.exists('file://' + chunkedFile)) { + await RNFS.unlink(chunkedFile); + } + await RNFS.copyFile(TEST_FILE, chunkedFile); + + // A small min and max, so the 1MB test file still splits into some parts. + // Production callers use the server's real part-size limits. + const {size} = await RNFS.stat(chunkedFile); + const ranges = Upload.chunkPlan(size, {min: 128 * 1024, max: 256 * 1024}); + + const uploadOpts: ChunkedUploadOptions = { + type: 'chunked', + id: 'chunked-demo', + path: chunkedFile, + parts: ranges.map((range, i) => ({ + url: `${CHUNKED_UPLOAD_URL}?partNum=${i + 1}`, + headers: { + 'Content-Type': 'application/octet-stream', + 'Content-Range': `bytes ${range.start}-${range.end - 1}/${size}`, + }, + range, + })), + expiresAt: Date.now() + 24 * 60 * 60 * 1000, + }; + + Upload.startUpload(uploadOpts) + .then(uploadId => { + console.log( + `Chunked upload started: ${uploadId} (${ranges.length} parts)`, + ); + setUploadId(uploadId); + setProgress(0); + }) + .catch(function (err) { + setUploadId(undefined); + setProgress(undefined); + console.log('Chunked upload error!', err); + }); + }; + return ( <> @@ -126,6 +180,7 @@ const App = () => {