diff --git a/.changeset/salty-knives-cut.md b/.changeset/salty-knives-cut.md new file mode 100644 index 000000000..b1c276abd --- /dev/null +++ b/.changeset/salty-knives-cut.md @@ -0,0 +1,6 @@ +--- +'@powersync/service-module-mongodb-storage': patch +'@powersync/service-core': patch +--- + +Remove global replication lock for MongoDB storage. diff --git a/docs/storage/mongodb-replication-fencing.md b/docs/storage/mongodb-replication-fencing.md new file mode 100644 index 000000000..eaeac859f --- /dev/null +++ b/docs/storage/mongodb-replication-fencing.md @@ -0,0 +1,48 @@ +# MongoDB replication fencing + +A lease assigns a replication stream to one process. A **fence** checks that the process still owns that lease when it writes. This stops a stalled process from publishing after another process takes over. + +## How it works + +Creating a writer requires a replication lease. Each writer keeps the lease token it started with. Every flush transaction checks that token against `sync_rules.lock.id` and updates the stream's existing heartbeat (`last_keepalive_ts`). The heartbeat always changes, even when two updates happen in the same millisecond. + +Changing the stream document makes lease takeover conflict with the transaction. Either: + +- Takeover happens first: the old writer's fence fails, so its transaction cannot publish. +- The old writer fences first: takeover must wait for that transaction to commit or abort. + +The fence runs on **every flush**, not only on `writer.commit()`, which publishes a checkpoint. Checkpoint writes check ownership even when there are no pending rows. + +Snapshot progress, table resolution, and activation also use fenced transactions. Updates confined to the stream document, such as resume positions and stream snapshot state, check the lease and update the heartbeat atomically. + +Writers sharing a stream lease also share an in-process FIFO mutex. It covers fenced transactions through commit and retries, plus standalone metadata updates, so snapshot and streaming writers queue instead of conflicting locally. ID reservations happen outside the mutex. + +Different streams use different mutexes and documents, so neither admission nor fencing introduces a global lock. Database fencing remains necessary for writers in other processes. + +## Reserved operation IDs + +Reserving IDs happens outside the flush transaction. A stale process can still reserve IDs, but that does not let it publish them. + +For example: + +1. Process 1 stalls and loses its lease to process 2. +2. Process 2 reserves range A. +3. Process 1 reserves a higher range B. +4. Process 1 tries to flush B. Its lease check fails; none of those operations become visible. +5. Process 2 can safely write A. B is unused. + +If process 1 finishes a flush **before** takeover, process 2 must also avoid writing below those persisted IDs. Every flush reads the stream's persisted head inside its fenced transaction and skips reserved IDs below that head. Legacy storage uses `max(last_checkpoint, keepalive_op)`; v3/v4 use `last_persisted_op`. + +Both rules matter: fence every flush, and allocate above the persisted head. Fencing only checkpoint commits would leave a consistency gap. + +## Limits + +- The fence checks the lease token, not its expiry time. Expiry permits takeover; it does not itself stop the old writer. Lease renewal failures also abort the local lease signal. +- Graceful shutdown lets the connector finish its current page and save progress. Losing the lease prevents further fenced writes. +- The fence orders transactions within a stream. The connector must still submit source changes in the correct order. +- A writer that loses its lease cannot write again, even after the successor releases its lease. A new job must acquire a lease and construct new writable storage. +- Setting a stream to `STOP` does not revoke its lease by itself. +- Full storage clearing, error reporting, collection creation/drop, post-commit cleanup, and external uploads are outside this fence. They rely on separate lifecycle or cleanup rules. In particular, clearing is not protected against a stalled cleanup process resuming after lease loss. +- Older service versions do not gain these ownership checks merely by sharing compatible storage. + +The main implementation is in [MongoSyncRulesLock](../../modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts) and [MongoBucketBatch](../../modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts). diff --git a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts index cf9167044..9d5074f93 100644 --- a/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/MongoBucketStorage.ts @@ -31,7 +31,9 @@ import { createMongoSyncBucketStorage } from './implementation/createMongoSyncBu import { PowerSyncMongo } from './implementation/db.js'; import { getMongoStorageConfig, StorageConfig, SyncRuleDocumentBase } from './implementation/models.js'; import { MongoChecksumOptions } from './implementation/MongoChecksums.js'; +import { MongoOpIdAllocator } from './implementation/MongoOpIdAllocator.js'; import { MongoPersistedReplicationStream } from './implementation/MongoPersistedReplicationStream.js'; +import { MongoSyncRulesLock } from './implementation/MongoSyncRulesLock.js'; import { stopReplicationStreamPipeline } from './implementation/SyncRuleStateUpdate.js'; import { SyncRuleDocumentV1 } from './implementation/v1/models.js'; import { ObjectStorage } from './implementation/v3/object-storage/ObjectStorage.js'; @@ -69,6 +71,25 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { private readonly client: mongo.MongoClient; public readonly replicationStreamNamePrefix: string; + private readonly opIdAllocators = new Map(); + + discardOpIdAllocator(streamId: number) { + this.opIdAllocators.get(streamId)?.allocator.discard(); + this.opIdAllocators.delete(streamId); + } + + getOpIdAllocator(stream: MongoPersistedReplicationStream, lock: MongoSyncRulesLock): MongoOpIdAllocator { + lock.throwIfAborted(); + const previous = this.opIdAllocators.get(stream.replicationStreamId); + if (previous?.lock === lock) { + return previous.allocator; + } + previous?.allocator.discard(); + const allocator = new MongoOpIdAllocator(this.db.versioned(stream.getStorageConfig())); + this.opIdAllocators.set(stream.replicationStreamId, { lock, allocator }); + return allocator; + } + private activeStorageCache: MongoSyncBucketStorage | undefined; public readonly db: PowerSyncMongo; @@ -91,10 +112,6 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { this.replicationStreamNamePrefix = options.replicationStreamNamePrefix; } - async [Symbol.asyncDispose]() { - // No-op - } - getInstance( replicationStream: storage.PersistedReplicationStream, options?: GetIntanceOptions @@ -107,6 +124,15 @@ export class MongoBucketStorage extends storage.BucketStorageFactory { replicationStreamId = Number(replicationStreamId); } const storageConfig = replicationStream.getStorageConfig(); + if (options?.replicationLock != null) { + if ( + !(options.replicationLock instanceof MongoSyncRulesLock) || + options.replicationLock.sync_rules_id !== replicationStream.replicationStreamId + ) { + throw new ReplicationAssertionError('Replication lock does not belong to this MongoDB stream'); + } + replicationStream.current_lock = options.replicationLock; + } const syncRuleStorage = createMongoSyncBucketStorage( this, replicationStreamId, diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts index 36323f516..373969064 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoBucketBatch.ts @@ -14,6 +14,7 @@ import { ErrorCode, errors, Logger, + ReplicationAbortedError, ReplicationAssertionError, ServiceError } from '@powersync/lib-services-framework'; @@ -33,22 +34,20 @@ import { mongoTableId } from '../../utils/util.js'; import { PersistedBatch } from './common/PersistedBatch.js'; import { LoadedSourceRecord, SourceRecordStore } from './common/SourceRecordStore.js'; import type { VersionedPowerSyncMongo } from './db.js'; +import { SyncRuleDocumentBase } from './models.js'; import { MAX_ROW_SIZE } from './MongoBucketBatchShared.js'; -import { MongoIdSequence } from './MongoIdSequence.js'; +import { MongoIdSequence, OpIdRangeExhausted } from './MongoIdSequence.js'; +import { MongoOpIdAllocator } from './MongoOpIdAllocator.js'; import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js'; +import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { MongoWriteBatch } from './MongoWriteBatch.js'; import { OperationBatch, RecordOperation } from './OperationBatch.js'; import { ObjectStorage } from './v3/object-storage/ObjectStorage.js'; import { createObjectStorageUsageWriterId } from './v3/object-storage/ObjectStorageUsage.js'; -// Currently, we can only have a single flush() at a time, since it locks the op_id sequence. -// While the MongoDB transaction retry mechanism handles this okay, using an in-process Mutex -// makes it more fair and has less overhead. -// -// In the future, we can investigate allowing multiple replication streams operating independently. -const replicationMutex = new utils.Mutex(); - export interface MongoBucketBatchOptions { + replicationLock: MongoSyncRulesLock; + opIdAllocator: MongoOpIdAllocator; db: VersionedPowerSyncMongo; /** * The parsed sync config set for this batch. @@ -145,7 +144,13 @@ export abstract class MongoBucketBatch constructor(options: MongoBucketBatchOptions) { super(); this.logger = options.logger; - this.options = options; + this.options = { + ...options, + signal: + options.signal == null + ? options.replicationLock.signal + : AbortSignal.any([options.signal, options.replicationLock.signal]) + }; this.client = options.db.client; this.db = options.db; this.replicationStreamId = options.replicationStreamId; @@ -215,7 +220,34 @@ export abstract class MongoBucketBatch return result; } - private async flushInner(options?: storage.BatchBucketFlushOptions): Promise { + /** Publish the checkpoint in the final flush transaction, or use the empty-commit path. */ + protected async flushAndCommit( + checkpoint: (stream: SyncRuleDocumentBase, lastOp: InternalOpId) => Promise, + commitWithoutFlush: () => Promise, + options?: storage.BatchBucketFlushOptions, + projection?: mongo.Document + ): Promise { + if (this.batch == null && this.write_checkpoint_batch.length === 0) { + return commitWithoutFlush(); + } + let result!: T; + while (this.batch != null || this.write_checkpoint_batch.length > 0) { + await this.flushInner( + options, + async (stream, lastOp) => { + result = await checkpoint(stream, lastOp); + }, + projection + ); + } + return result; + } + + private async flushInner( + options?: storage.BatchBucketFlushOptions, + checkpoint?: (stream: SyncRuleDocumentBase, lastOp: InternalOpId) => Promise, + projection?: mongo.Document + ): Promise { const batch = this.batch; let last_op: InternalOpId | null = null; let resumeBatch: OperationBatch | null = null; @@ -229,21 +261,33 @@ export abstract class MongoBucketBatch await this.prepareCustomWriteCheckpoints(); } - await this.withReplicationTransaction(`Flushing ${batch?.length ?? 0} ops`, async (session, opSeq) => { - clearedError = false; - if (batch != null) { - const result = await this.replicateBatch(session, batch, opSeq, options); - resumeBatch = result.resumeBatch; - clearedError ||= result.clearedError; - } + await this.withReplicationTransaction( + `Flushing ${batch?.length ?? 0} ops`, + async (session, opSeq) => { + clearedError = false; + if (batch != null) { + const result = await this.replicateBatch(session, batch, opSeq, options); + resumeBatch = result.resumeBatch; + clearedError ||= result.clearedError; + } - if (this.write_checkpoint_batch.length > 0) { - this.logger.info(`Writing ${this.write_checkpoint_batch.length} custom write checkpoints`); - await this.batchCreateCustomWriteCheckpoints(session, opSeq.next()); - } + if (this.write_checkpoint_batch.length > 0) { + this.logger.info(`Writing ${this.write_checkpoint_batch.length} custom write checkpoints`); + await this.batchCreateCustomWriteCheckpoints(session, opSeq.next()); + } - last_op = opSeq.last(); - }); + last_op = opSeq.last(); + }, + async (stream, lastOp) => { + if (checkpoint != null && resumeBatch == null) { + // The checkpoint must also persist the head, including when visibility is blocked. + await checkpoint(stream, lastOp); + return true; + } + return false; + }, + projection + ); // Keep checkpoints available if the transaction retries after writing them. this.write_checkpoint_batch = []; @@ -643,17 +687,46 @@ export abstract class MongoBucketBatch return result; } - protected async withTransaction(cb: () => Promise) { - using lockSpan = this.tracer.span('storage', 'internal_lock'); - await replicationMutex.exclusiveLock(async () => { - lockSpan.end(); - await this.session.withTransaction( + protected async fence(session = this.session, projection?: mongo.Document) { + // Graceful cancellation belongs to the source connector's page/batch boundary. + // It must still be able to persist progress for rows already flushed. Lease + // loss is different: the fence rejects every subsequent writer transaction. + return MongoSyncRulesLock.fence( + this.db, + this.replicationStreamId, + this.options.replicationLock, + session, + projection + ); + } + + protected async withTransaction( + cb: (stream: SyncRuleDocumentBase) => Promise, + session = this.session + ): Promise { + return this.withFencedTransaction(() => this.fence(session), cb, session); + } + + /** The first operation must modify the stream document conditional on lease ownership. */ + protected async withFencedTransaction( + acquireFence: () => Promise, + cb: (stream: S) => Promise, + session = this.session + ): Promise { + return this.withWriterLock(() => + session.withTransaction( async () => { + const stream = await acquireFence(); try { - await cb(); + const result = await cb(stream); + this.options.replicationLock.throwIfAborted(); + return result; } catch (e: unknown) { + if (e instanceof OpIdRangeExhausted) { + throw e; + } if (e instanceof mongo.MongoError && e.hasErrorLabel('TransientTransactionError')) { - // Likely write conflict caused by concurrent write stream replicating + // Likely write conflict caused by concurrent writes to this replication stream. } else { this.logger.warn('Transaction error', e as Error); } @@ -663,102 +736,118 @@ export abstract class MongoBucketBatch throw e; } }, - { maxCommitTimeMS: 10000 } - ); + { maxCommitTimeMS: 10000, writeConcern: { w: 'majority' } } + ) + ); + } + + private async withWriterLock(callback: () => Promise): Promise { + using lockSpan = this.tracer.span('storage', 'internal_lock'); + return this.options.replicationLock.writerMutex.exclusiveLock(async () => { + lockSpan.end(); + this.options.replicationLock.throwIfAborted(); + return callback(); }); } - private async withReplicationTransaction( - description: string, - callback: (session: mongo.ClientSession, opSeq: MongoIdSequence) => Promise + /** Single-document updates enforce ownership atomically, without a separate transaction. */ + protected async updateStreamMetadata( + set: mongo.Document, + filter: mongo.Document = {}, + writeConcern: mongo.WriteConcernSettings = { w: 'majority' } ): Promise { - let flushTry = 0; - - const start = Date.now(); - const lastTry = start + 90000; - - const session = this.session; - - await this.withTransaction(async () => { - flushTry += 1; - if (flushTry % 10 == 0) { - this.logger.info(`${description} - try ${flushTry}`); - } - if (flushTry > 20 && Date.now() > lastTry) { - throw new ServiceError(ErrorCode.PSYNC_S1402, 'Max transaction tries exceeded'); - } - - const next_op_id_doc = await this.db.op_id_sequence.findOneAndUpdate( - { - _id: 'main' - }, - { - $setOnInsert: { op_id: 0n }, - $set: { - // Force update to ensure we get a mongo lock - ts: Date.now() - } - }, - { - upsert: true, - returnDocument: 'after', - session - } - ); - const opSeq = new MongoIdSequence(next_op_id_doc?.op_id ?? 0n); + await this.withWriterLock(() => this.writeStreamMetadata(set, filter, writeConcern)); + } - await callback(session, opSeq); + /** Caller must already be inside a fenced transaction holding the writer mutex. */ + protected async updateStreamMetadataInTransaction(set: mongo.Document, filter: mongo.Document = {}): Promise { + await this.writeStreamMetadata(set, filter); + } - const writes = this.db.createWriteBatch(session, { ordered: false }); - writes.updateOne( - this.db.op_id_sequence, - { - _id: 'main' - }, - { - $set: { - op_id: opSeq.last() - } - } - ); + private async writeStreamMetadata( + set: mongo.Document, + filter: mongo.Document, + writeConcern?: mongo.WriteConcernSettings + ): Promise { + const result = await this.db.sync_rules.updateOne( + { ...filter, ...MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock) }, + [{ $set: { ...set, ...MongoSyncRulesLock.heartbeatUpdate() } }], + { session: this.session, ...(writeConcern == null ? {} : { writeConcern }) } + ); + if (result.matchedCount === 0) { + throw new ReplicationAbortedError('Replication writer no longer owns the stream'); + } + } - writes.updateOne( - this.db.sync_rules, - { - _id: this.replicationStreamId - }, - { - $set: { - last_keepalive_ts: new Date() + private async withReplicationTransaction( + description: string, + callback: (session: mongo.ClientSession, opSeq: MongoIdSequence) => Promise, + finish?: (stream: SyncRuleDocumentBase, lastOp: InternalOpId) => Promise, + projection?: mongo.Document + ): Promise { + let flushTry = 0; + const lastTry = Date.now() + 90000; + const allocator = this.options.opIdAllocator; + let lastOp = 0n; + // Refill outside the publication transaction, before evaluating any rows. + // Exhaustion remains a fallback for large batches or a newer stream head. + this.options.replicationLock.throwIfAborted(); + await allocator.ensureCapacity(); + for (;;) { + try { + await this.withFencedTransaction( + () => this.fence(this.session, projection), + async (stream) => { + flushTry += 1; + if (flushTry % 10 == 0) { + this.logger.info(`${description} - try ${flushTry}`); + } + if (flushTry > 20 && Date.now() > lastTry) { + throw new ServiceError(ErrorCode.PSYNC_S1402, 'Max transaction tries exceeded'); + } + // The fence already holds this stream document for the transaction. + // A different writer may have used higher IDs since our last flush. + const opSeq = allocator.sequence(this.persistedOpHead(stream)); + await callback(this.session, opSeq); + lastOp = opSeq.last(); + + if (!(await finish?.(stream, lastOp))) { + const writes = this.db.createWriteBatch(this.session, { ordered: false }); + // Allow subclasses to persist additional flush-time state in the same transaction. + this.onReplicationTransactionFlush(writes, lastOp); + await writes.execute(); + } + // Notifications and cleanup must wait until the transaction commits. } + ); + allocator.committed(lastOp); + return; + } catch (error) { + if (!(error instanceof OpIdRangeExhausted)) { + throw error; } - ); - - // Allow subclasses to persist additional flush-time state in the same transaction - // (e.g. v3 advances the stream-level last_persisted_op). - this.onReplicationTransactionFlush(writes, opSeq.last()); - await writes.execute(); - // We don't notify checkpoint here - we don't make any checkpoint updates directly - }); + // withTransaction has aborted every write. Reserve outside that transaction, + // then replay evaluation from the same committed stream head. Existing + // ranges can be reused on retry because no operation from this attempt committed. + this.options.replicationLock.throwIfAborted(); + await allocator.reserve(); + } + } } - /** - * Hook called inside the replication flush transaction, after ops have been persisted. - * - * The base implementation does nothing; v3 storage overrides this to `$max` the stream-level - * `last_persisted_op` durably within the same transaction. - */ - protected onReplicationTransactionFlush(_writes: MongoWriteBatch, _lastOp: InternalOpId): void { - // No-op by default (v1 behaviour unchanged). - } + /** Highest durably persisted operation, including operations not yet checkpointed. */ + protected abstract persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId; + + /** Advance the persisted head in the same transaction as the operations. */ + protected abstract onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void; /** * Called after a replication transaction has successfully committed, with the last persisted op id. * * v1 storage tracks this in memory to fold into the next checkpoint. v3 storage does not need it: * the stream-level `last_persisted_op` is already advanced durably by - * {@link onReplicationTransactionFlush} within the same transaction, and checkpoints read it - * from the document. + * {@link onReplicationTransactionFlush} or the combined checkpoint update within the same + * transaction, and empty commits read it from the document. * * Keep calls to this adjacent to {@link withReplicationTransaction} usage - both must observe * every path that persists ops. @@ -837,7 +926,9 @@ export abstract class MongoBucketBatch await this.withTransaction(async () => { for (let table of sourceTables) { - await this.db.commonSourceTables(this.replicationStreamId).deleteOne({ _id: mongoTableId(table.id) }); + await this.db + .commonSourceTables(this.replicationStreamId) + .deleteOne({ _id: mongoTableId(table.id) }, { session: this.session }); } }); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoIdSequence.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoIdSequence.ts index 633d15cfe..6a0497ef5 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoIdSequence.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoIdSequence.ts @@ -9,7 +9,10 @@ import { ReplicationAssertionError } from '@powersync/lib-services-framework'; export class MongoIdSequence { private _last: bigint; - constructor(last: bigint) { + constructor( + last: bigint, + private readonly ranges?: readonly OpIdRange[] + ) { if (typeof last != 'bigint') { throw new ReplicationAssertionError(`BigInt required, got ${last} ${typeof last}`); } @@ -17,6 +20,14 @@ export class MongoIdSequence { } next() { + if (this.ranges != null) { + const next = this.ranges.find((range) => range.end > this._last); + if (next == null) { + throw new OpIdRangeExhausted(); + } + this._last = this._last + 1n < next.start ? next.start : this._last + 1n; + return this._last; + } return ++this._last; } @@ -24,3 +35,12 @@ export class MongoIdSequence { return this._last; } } + +/** Inclusive, durably reserved operation IDs. Gaps between ranges are valid. */ +export interface OpIdRange { + start: bigint; + end: bigint; +} + +/** Abort the transaction before reserving more IDs and retrying its evaluation. */ +export class OpIdRangeExhausted extends Error {} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoOpIdAllocator.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoOpIdAllocator.ts new file mode 100644 index 000000000..2d01a6621 --- /dev/null +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoOpIdAllocator.ts @@ -0,0 +1,105 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { ReplicationAssertionError } from '@powersync/lib-services-framework'; +import { VersionedPowerSyncMongo } from './db.js'; +import { MongoIdSequence, OpIdRange } from './MongoIdSequence.js'; + +const RESERVATION_SIZE = 65_536n; +const LOW_WATER_MARK = 16_384n; +const MAX_OP_ID = (1n << 63n) - 1n; + +/** + * Shared by writers of one stream and lease owner in this process. Reservations + * use short majority writes, never the publication session. A crashed owner + * abandons its unused ranges; the global sequence is an allocation watermark. + */ +export class MongoOpIdAllocator { + private ranges: OpIdRange[] = []; + private reserving?: Promise; + + constructor(private readonly db: VersionedPowerSyncMongo) {} + + discard() { + this.ranges = []; + } + + sequence(persistedOp: bigint): MongoIdSequence { + // The stream is fenced before reading this head. Another writer may have + // committed since this allocator was last used, including in another process. + // Never reuse reserved IDs below that durable head. + return new MongoIdSequence(persistedOp, this.ranges.slice()); + } + + committed(lastOp: bigint) { + this.ranges = this.ranges + .filter((range) => range.end > lastOp) + .map((range) => ({ start: range.start > lastOp ? range.start : lastOp + 1n, end: range.end })); + } + + /** Top up between transactions, retaining every unused ID in existing ranges. */ + async ensureCapacity(): Promise { + const remaining = this.ranges.reduce((total, range) => total + range.end - range.start + 1n, 0n); + if (remaining < LOW_WATER_MARK) { + await this.reserve(); + } + } + + async reserve(): Promise { + if (this.reserving != null) { + return this.reserving; + } + this.reserving = this.reserveRange(); + try { + await this.reserving; + } finally { + this.reserving = undefined; + } + } + + /** + * Atomically reserve a range of operation IDs in the global sequence. + * + * We continue using the range across multiple batches, until the range is exhausted, + * so we don't "waste" large segements in long-running processes with small batches. + * However, we can't return unused portions back to the database if we don't use them. + */ + private async reserveRange(): Promise { + try { + // Return the previous watermark so an unchanged value at the limit can be + // distinguished from a successful reservation ending near the limit. + const previous = await this.db.op_id_sequence.findOneAndUpdate( + { _id: 'main' }, + [ + { + $set: { + op_id: { + $let: { + vars: { current: { $ifNull: ['$op_id', 0n] } }, + in: { + $cond: [ + { $lte: ['$$current', MAX_OP_ID - RESERVATION_SIZE] }, + { $add: ['$$current', RESERVATION_SIZE] }, + '$$current' + ] + } + } + } + } + } + ], + { upsert: true, returnDocument: 'before', writeConcern: { w: 'majority' } } + ); + const last = previous?.op_id ?? 0n; + if (last > MAX_OP_ID - RESERVATION_SIZE) { + // Not expected to ever happen, unless the sequence was manually modified + throw new ReplicationAssertionError('Operation ID sequence exhausted'); + } + this.ranges.push({ start: last + 1n, end: last + RESERVATION_SIZE }); + } catch (error) { + // Concurrent first reservations may race to insert the sequence document - retry in that case. + if (error instanceof mongo.MongoServerError && error.code === 11000) { + return this.reserveRange(); + } + throw error; + } + } +} diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts index d3fd8354f..3b5cf4a42 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncBucketStorage.ts @@ -43,6 +43,7 @@ import { MongoCompactOptions, MongoCompactor } from './MongoCompactor.js'; import { MongoParameterCompactor } from './MongoParameterCompactor.js'; import { MongoParsedSyncConfigSet } from './MongoParsedSyncConfigSet.js'; import { MongoPersistedReplicationStream } from './MongoPersistedReplicationStream.js'; +import { MongoSyncRulesLock } from './MongoSyncRulesLock.js'; import { MongoCheckpointAPIOptions, MongoWriteCheckpointAPI } from './MongoWriteCheckpointAPI.js'; import { ObjectStorage } from './v3/object-storage/ObjectStorage.js'; @@ -109,6 +110,10 @@ export abstract class MongoSyncBucketStorage public readonly readPreference: mongo.ReadPreference | undefined; public readonly clearBatchThrottleRate: number; #storageInitialized = false; + /** + * Required for writing, optional for reading. + */ + private readonly replicationLock: MongoSyncRulesLock | null; constructor( public readonly factory: MongoBucketStorage, @@ -119,6 +124,7 @@ export abstract class MongoSyncBucketStorage options: MongoSyncBucketStorageOptions ) { super(); + this.replicationLock = replicationStream.current_lock; this.storageConfig = options.storageConfig; this.objectStorage = options.objectStorage; // Keep small chunks inline in MongoDB rather than offloading them to S3. @@ -269,12 +275,15 @@ export abstract class MongoSyncBucketStorage * The version-independent part of the batch options. */ protected writerBatchOptions(options: storage.CreateWriterOptions): Omit { + const replicationLock = this.requireReplicationLock(); return { logger: options.logger ?? this.logger, db: this.db, parsedSyncConfig: this.getParsedSyncConfigSet(options), replicationStreamId: this.replicationStreamId, replicationStreamName: this.replicationStreamName, + replicationLock, + opIdAllocator: this.factory.getOpIdAllocator(this.replicationStream, replicationLock), storeCurrentData: options.storeCurrentData, skipExistingRows: options.skipExistingRows ?? false, markRecordUnavailable: options.markRecordUnavailable, @@ -286,7 +295,16 @@ export abstract class MongoSyncBucketStorage }; } + private requireReplicationLock(): MongoSyncRulesLock { + if (this.replicationLock == null) { + throw new ServiceAssertionError('A replication lease is required to create a writer'); + } + this.replicationLock.throwIfAborted(); + return this.replicationLock; + } + async createWriter(options: storage.CreateWriterOptions): Promise { + this.requireReplicationLock(); await this.initializeStorage(); const writer = await this.createWriterImpl(options); @@ -385,6 +403,7 @@ export abstract class MongoSyncBucketStorage throw new ReplicationAbortedError('Aborted clearing data', signal.reason); } + this.factory.discardOpIdAllocator(this.replicationStreamId); await this.clearSyncRuleState(); await this.clearBucketData(signal); diff --git a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts index 527eeda9d..13dc12c00 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/MongoSyncRulesLock.ts @@ -1,8 +1,8 @@ import crypto from 'crypto'; import { mongo } from '@powersync/lib-service-mongodb'; -import { ErrorCode, Logger, ServiceError } from '@powersync/lib-services-framework'; -import { storage } from '@powersync/service-core'; +import { ErrorCode, Logger, ReplicationAbortedError, ServiceError } from '@powersync/lib-services-framework'; +import { storage, utils } from '@powersync/service-core'; import { VersionedPowerSyncMongo } from './db.js'; const LOCK_DURATION_MS = 60 * 1000; @@ -12,6 +12,12 @@ const LOCK_DURATION_MS = 60 * 1000; * processes that replication stream at a time. */ export class MongoSyncRulesLock implements storage.ReplicationLock { + /** FIFO admission shared by this lease's writers; database fencing still enforces ownership. */ + readonly writerMutex = new utils.Mutex(); + + private readonly abort = new AbortController(); + readonly signal = this.abort.signal; + private readonly refreshInterval: NodeJS.Timeout; /** @@ -67,13 +73,14 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { constructor( private db: VersionedPowerSyncMongo, public sync_rules_id: number, - private lock_id: string, + public readonly lock_id: string, private logger: Logger ) { this.refreshInterval = setInterval(async () => { try { await this.refresh(); } catch (e) { + this.abort.abort(e); this.logger.error('Failed to refresh lock', e); clearInterval(this.refreshInterval); } @@ -81,6 +88,7 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { } async release(): Promise { + this.abort.abort(new Error('Replication lock released')); clearInterval(this.refreshInterval); const result = await this.db.sync_rules.updateOne( { @@ -97,6 +105,51 @@ export class MongoSyncRulesLock implements storage.ReplicationLock { } } + throwIfAborted(): void { + this.signal.throwIfAborted(); + } + + static ownerFilter(streamId: number, lock: MongoSyncRulesLock) { + lock.throwIfAborted(); + return { _id: streamId, 'lock.id': lock.lock_id }; + } + + static heartbeatUpdate() { + // Always change the existing heartbeat, even for writes in the same + // millisecond. A no-op update is not sufficient for a transactional fence. + return { + last_keepalive_ts: { + $max: ['$$NOW', { $add: [{ $ifNull: ['$last_keepalive_ts', new Date(0)] }, 1] }] + } + }; + } + + static assertOwned(document: T | null): T { + if (document == null) { + throw new ReplicationAbortedError('Replication writer no longer owns the stream'); + } + return document; + } + + /** + * A write, not just an ownership read: takeover conflicts with every transaction + * that publishes under this owner. Every writer must hold a stream lease. + */ + static async fence( + db: VersionedPowerSyncMongo, + streamId: number, + lock: MongoSyncRulesLock, + session: mongo.ClientSession, + projection: mongo.Document = { last_persisted_op: 1, last_checkpoint: 1, keepalive_op: 1 } + ) { + const doc = await db.sync_rules.findOneAndUpdate( + this.ownerFilter(streamId, lock), + [{ $set: this.heartbeatUpdate() }], + { session, returnDocument: 'after', projection } + ); + return this.assertOwned(doc); + } + private async refresh(): Promise { const result = await this.db.sync_rules.findOneAndUpdate( { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts index bc20a9ef2..f09490fdc 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoBucketBatchV1.ts @@ -6,8 +6,11 @@ import * as bson from 'bson'; import { mongoTableId } from '../../../utils/util.js'; import { calculateCheckpointState } from '../CheckpointState.js'; import { MongoBucketBatch, MongoBucketBatchOptions } from '../MongoBucketBatch.js'; +import { MongoSyncRulesLock } from '../MongoSyncRulesLock.js'; +import { MongoWriteBatch } from '../MongoWriteBatch.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; import { SourceRecordStore } from '../common/SourceRecordStore.js'; +import { SyncRuleDocumentBase } from '../models.js'; import { PersistedBatchV1 } from './PersistedBatchV1.js'; import { SourceRecordStoreV1 } from './SourceRecordStoreV1.js'; import { VersionedPowerSyncMongoV1 } from './VersionedPowerSyncMongoV1.js'; @@ -34,6 +37,27 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { this.store = new SourceRecordStoreV1(this.db, this.replicationStreamId); } + protected override persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId { + const legacy = stream as SyncRuleDocumentV1; + const checkpoint = legacy.last_checkpoint ?? 0n; + const keepalive = legacy.keepalive_op == null ? 0n : BigInt(legacy.keepalive_op); + return checkpoint > keepalive ? checkpoint : keepalive; + } + + protected override onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void { + // Keep flushed operations recoverable before a checkpoint can be published. + // keepalive_op remains a decimal string for compatibility with legacy readers. + writes.updateOne(this.db.sync_rules, { _id: this.replicationStreamId }, [ + { + $set: { + keepalive_op: { + $toString: { $max: [{ $toLong: '$keepalive_op' }, '$last_checkpoint', lastOp] } + } + } + } + ]); + } + protected override recordPersistedOp(lastOp: InternalOpId): void { this.persisted_op = lastOp; } @@ -101,7 +125,8 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { })); let result: storage.ResolveTablesResult | null = null; - await this.db.client.withSession(async (session) => { + const session = this.session; + await this.withTransaction(async () => { const col = this.db.sourceTablesV1(this.replicationStreamId); // Find records that overlap by name or relation id. @@ -219,129 +244,136 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options }; - await this.flush(options); - using _ = this.tracer.span('storage', 'commit'); - const now = new Date(); + const updateCheckpoint = async (persistedOp = this.persisted_op) => { + const now = new Date(); - await this.db.write_checkpoints.updateMany( - { - processed_at_lsn: null, - 'lsns.1': { $lte: lsn } - }, - { - $set: { - processed_at_lsn: lsn - } - }, - { - session: this.session - } - ); + const can_checkpoint = { + $and: [ + { $eq: ['$snapshot_done', true] }, + { + $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }] + }, + { + $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }] + } + ] + }; - const can_checkpoint = { - $and: [ - { $eq: ['$snapshot_done', true] }, - { - $or: [{ $eq: ['$last_checkpoint_lsn', null] }, { $lte: ['$last_checkpoint_lsn', { $literal: lsn }] }] - }, - { - $or: [{ $eq: ['$no_checkpoint_before', null] }, { $lte: ['$no_checkpoint_before', { $literal: lsn }] }] - } - ] - }; + const new_keepalive_op = { + $cond: [ + can_checkpoint, + { $literal: null }, + { + $toString: { + $max: [{ $toLong: '$keepalive_op' }, { $literal: persistedOp }, 0n] + } + } + ] + }; + + const new_last_checkpoint = { + $cond: [ + can_checkpoint, + { + $max: ['$last_checkpoint', { $literal: persistedOp }, { $toLong: '$keepalive_op' }, 0n] + }, + '$last_checkpoint' + ] + }; - const new_keepalive_op = { - $cond: [ - can_checkpoint, - { $literal: null }, + const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate( + MongoSyncRulesLock.ownerFilter(this.replicationStreamId, this.options.replicationLock), + [ + { + $set: { + _can_checkpoint: can_checkpoint, + _not_empty: createEmptyCheckpoints + ? true + : { + $or: [ + { $literal: createEmptyCheckpoints }, + { $ne: ['$keepalive_op', new_keepalive_op] }, + { $ne: ['$last_checkpoint', new_last_checkpoint] } + ] + } + } + }, + { + $set: { + ...MongoSyncRulesLock.heartbeatUpdate(), + last_checkpoint_lsn: { + $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn'] + }, + last_checkpoint_ts: { + $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts'] + }, + + last_fatal_error: { $literal: null }, + last_fatal_error_ts: { $literal: null }, + keepalive_op: new_keepalive_op, + last_checkpoint: new_last_checkpoint, + snapshot_lsn: { + $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn'] + } + } + }, + { + $unset: ['_can_checkpoint', '_not_empty'] + } + ], { - $toString: { - $max: [{ $toLong: '$keepalive_op' }, { $literal: this.persisted_op }, 0n] + session: this.session, + returnDocument: 'before', + projection: { + snapshot_done: 1, + last_checkpoint_lsn: 1, + no_checkpoint_before: 1, + keepalive_op: 1, + last_checkpoint: 1 } } - ] - }; + )) as SyncRuleDocumentV1; - const new_last_checkpoint = { - $cond: [ - can_checkpoint, - { - $max: ['$last_checkpoint', { $literal: this.persisted_op }, { $toLong: '$keepalive_op' }, 0n] - }, - '$last_checkpoint' - ] + return MongoSyncRulesLock.assertOwned(preUpdateDocument); }; - const preUpdateDocument = (await this.db.sync_rules.findOneAndUpdate( - { _id: this.replicationStreamId }, - [ + const finishCheckpoint = async (preUpdateDocument: SyncRuleDocumentV1, lastOp?: InternalOpId) => { + await this.db.write_checkpoints.updateMany( { - $set: { - _can_checkpoint: can_checkpoint, - _not_empty: createEmptyCheckpoints - ? true - : { - $or: [ - { $literal: createEmptyCheckpoints }, - { $ne: ['$keepalive_op', new_keepalive_op] }, - { $ne: ['$last_checkpoint', new_last_checkpoint] } - ] - } - } + processed_at_lsn: null, + 'lsns.1': { $lte: lsn } }, { $set: { - last_checkpoint_lsn: { - $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: lsn }, '$last_checkpoint_lsn'] - }, - last_checkpoint_ts: { - $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: now }, '$last_checkpoint_ts'] - }, - last_keepalive_ts: { $literal: now }, - last_fatal_error: { $literal: null }, - last_fatal_error_ts: { $literal: null }, - keepalive_op: new_keepalive_op, - last_checkpoint: new_last_checkpoint, - snapshot_lsn: { - $cond: [{ $and: ['$_can_checkpoint', '$_not_empty'] }, { $literal: null }, '$snapshot_lsn'] - } + processed_at_lsn: lsn } }, { - $unset: ['_can_checkpoint', '_not_empty'] + session: this.session } - ], - { - session: this.session, - returnDocument: 'before', - projection: { - snapshot_done: 1, - last_checkpoint_lsn: 1, - no_checkpoint_before: 1, - keepalive_op: 1, - last_checkpoint: 1 - } - } - )) as SyncRuleDocumentV1; - - if (preUpdateDocument == null) { - throw new ReplicationAssertionError( - 'Failed to update checkpoint - no matching sync_rules document for _id: ' + this.replicationStreamId ); - } - const checkpointState = calculateCheckpointState({ - lsn, - snapshotDone: preUpdateDocument.snapshot_done === true, - lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn, - noCheckpointBefore: preUpdateDocument.no_checkpoint_before, - keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op), - lastCheckpoint: preUpdateDocument.last_checkpoint, - persistedOp: this.persisted_op, - createEmptyCheckpoints - }); + const checkpointState = calculateCheckpointState({ + lsn, + snapshotDone: preUpdateDocument.snapshot_done === true, + lastCheckpointLsn: preUpdateDocument.last_checkpoint_lsn, + noCheckpointBefore: preUpdateDocument.no_checkpoint_before, + keepaliveOp: preUpdateDocument.keepalive_op == null ? null : BigInt(preUpdateDocument.keepalive_op), + lastCheckpoint: preUpdateDocument.last_checkpoint, + persistedOp: lastOp ?? this.persistedOpHead(preUpdateDocument), + createEmptyCheckpoints + }); + return { checkpointState, preUpdateDocument }; + }; + + const { checkpointState, preUpdateDocument } = await this.flushAndCommit( + async (_stream, lastOp) => finishCheckpoint(await updateCheckpoint(lastOp), lastOp), + // Without a flush, the checkpoint update itself fences the transaction. + () => this.withFencedTransaction(updateCheckpoint, finishCheckpoint), + options + ); if (checkpointState.checkpointBlocked) { if (Date.now() - this.lastWaitingLogThrottled > 5_000) { this.logger.info( @@ -379,41 +411,20 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { async setResumeLsn(lsn: string): Promise { using _ = this.tracer.span('storage', 'set_resume_lsn'); - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId - }, - { - $set: { - snapshot_lsn: lsn - } - }, - { - session: this.session, - // Losing occasional resume LSN is fine. That may mean reprocessing - // some source changes in some edge cases, which is not an issue since - // changes are processed in an idempotent way. - writeConcern: { w: 1 } - } - ); + // Losing occasional resume LSN would only reprocess source changes. + // Keep the lease check atomic, but do not wait for majority replication of this resume hint. + await this.updateStreamMetadata({ snapshot_lsn: { $literal: lsn } }, {}, { w: 1 }); } async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise { - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId - }, - { - $set: { - snapshot_done: true, - last_keepalive_ts: new Date() - }, - $max: { - no_checkpoint_before: no_checkpoint_before_lsn - } - }, - { session: this.session } - ); + await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn)); + } + + private snapshotDoneUpdate(no_checkpoint_before_lsn: string) { + return { + snapshot_done: true, + no_checkpoint_before: { $max: ['$no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] } + }; } async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise { @@ -443,22 +454,12 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { } } - await this.markAllSnapshotDone(no_checkpoint_before_lsn); + await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn)); }); } async markTableSnapshotRequired(_table: storage.SourceTable): Promise { - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId - }, - { - $set: { - snapshot_done: false - } - }, - { session: this.session } - ); + await this.updateStreamMetadata({ snapshot_done: false }); } async markTableSnapshotDone( @@ -490,9 +491,6 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { _id: this.replicationStreamId }, { - $set: { - last_keepalive_ts: new Date() - }, $max: { no_checkpoint_before: no_checkpoint_before_lsn } @@ -516,7 +514,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { const session = this.session; let activated = false; let needsFutureActivationCheck = true; - await session.withTransaction(async () => { + await this.withTransaction(async () => { // Reset on transaction retries. activated = false; needsFutureActivationCheck = true; @@ -554,7 +552,7 @@ export class MongoBucketBatchV1 extends MongoBucketBatch { } else if (doc?.state != storage.SyncRuleState.PROCESSING) { needsFutureActivationCheck = false; } - }); + }, session); if (activated) { this.logger.info(`Activated new replication stream at ${lsn}`); await this.db.notifyCheckpoint(); diff --git a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts index 343a5fef5..299c033a2 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v1/MongoSyncBucketStorageV1.ts @@ -90,7 +90,7 @@ export class MongoSyncBucketStorageV1 extends MongoSyncBucketStorage { if (keepaliveOp == null && lastCheckpoint == null) { return null; } - return (keepaliveOp ?? 0n) > (lastCheckpoint ?? 0n) ? keepaliveOp : lastCheckpoint; + return [keepaliveOp ?? 0n, lastCheckpoint ?? 0n].reduce((a, b) => (a > b ? a : b)); } protected async createWriterImpl(options: storage.CreateWriterOptions): Promise { diff --git a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts index d45415b1b..09f592bad 100644 --- a/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts +++ b/modules/module-mongodb-storage/src/storage/implementation/v3/MongoBucketBatchV3.ts @@ -11,6 +11,7 @@ import { MongoWriteBatch } from '../MongoWriteBatch.js'; import { stopReplicationStreamPipeline } from '../SyncRuleStateUpdate.js'; import { PersistedBatch } from '../common/PersistedBatch.js'; import { SourceRecordStore } from '../common/SourceRecordStore.js'; +import { SyncRuleDocumentBase } from '../models.js'; import { PersistedBatchV3 } from './PersistedBatchV3.js'; import { SourceRecordStoreV3 } from './SourceRecordStoreV3.js'; import { VersionedPowerSyncMongoV3 } from './VersionedPowerSyncMongoV3.js'; @@ -61,7 +62,11 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { return this.store; } - protected override onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: bigint): void { + protected override persistedOpHead(stream: SyncRuleDocumentBase): InternalOpId { + return (stream as ReplicationStreamDocumentV3).last_persisted_op ?? 0n; + } + + protected override onReplicationTransactionFlush(writes: MongoWriteBatch, lastOp: InternalOpId): void { // Durably advance the stream-level head of persisted ops within the flush transaction. // This ensures a checkpoint created later (even by an empty commit, or by a freshly-appended // config that replicates nothing) covers all ops persisted before a potential crash. @@ -171,7 +176,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { const session = this.db.client.startSession(); await using _ = { [Symbol.asyncDispose]: () => session.endSession() }; - await session.withTransaction(async () => { + await this.withTransaction(async () => { const col = this.db.sourceTables(this.replicationStreamId); // Find records that overlap by name or relation id. @@ -241,7 +246,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { sourceTableFromDocument(doc, context.connectionTag, syncConfig, mapping, eventById) ) }; - }); + }, session); return result!; } @@ -266,156 +271,143 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { async commit(lsn: string, options?: storage.BucketBatchCommitOptions): Promise { const { createEmptyCheckpoints } = { ...storage.DEFAULT_BUCKET_BATCH_COMMIT_OPTIONS, ...options }; - await this.flush(options); - using _ = this.tracer.span('storage', 'commit'); - const now = new Date(); - - await this.db.write_checkpoints.updateMany( - { - processed_at_lsn: null, - 'lsns.1': { $lte: lsn } - }, - { - $set: { - processed_at_lsn: lsn - } - }, - { - session: this.session - } - ); - - const preUpdateDocument = (await this.db.sync_rules.findOne( - { - _id: this.replicationStreamId, - 'sync_configs._id': { $in: this.syncConfigIds } - }, - { - session: this.session, - projection: { - sync_configs: 1, - last_persisted_op: 1 - } - } - )) as ReplicationStreamDocumentV3 | null; - - const states = - preUpdateDocument?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? []; - if (states.length == 0) { - throw new ReplicationAssertionError( - `Failed to update checkpoint - no matching sync_config for _id: ${this.replicationStreamId}/${this.syncConfigIds - .map((id) => id.toHexString()) - .join(',')}` + const checkpoint = async (stream: SyncRuleDocumentBase, lastOp?: InternalOpId) => { + const now = new Date(); + const preUpdateDocument = stream as ReplicationStreamDocumentV3; + const writes = this.db.createWriteBatch(this.session, { ordered: false }); + writes.updateMany( + this.db.write_checkpoints, + { processed_at_lsn: null, 'lsns.1': { $lte: lsn } }, + { $set: { processed_at_lsn: lsn } } ); - } - // The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE) - // sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP) - // remain embedded in the document and are ignored here. - const missingSyncConfig = preUpdateDocument!.sync_configs.find( - (config) => - [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) && - !this.syncConfigIds.some((id) => id.equals(config._id)) - ); - if (missingSyncConfig != null) { - throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`); - } - // Effective head of the stream's op sequence. - // last_persisted_op is $max-advanced durably in the same transaction as every flush, and this - // read uses the same session, so it covers all ops persisted by this batch. - const newCheckpoint = - preUpdateDocument?.last_persisted_op == null ? 0n : BigInt(preUpdateDocument.last_persisted_op); - - let checkpointBlocked = false; - let checkpointCreated = false; - let checkpointLogState: unknown = null; - const unblockedConfigIds: bson.ObjectId[] = []; - - for (const state of states) { - if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) { - // last_persisted_op is $max-advanced durably in the same transaction as every flush, and - // checkpoints are only ever created at that head, so a checkpoint past the head means the - // op sequence or the stored state is corrupt. + const states = + preUpdateDocument?.sync_configs?.filter((config) => this.syncConfigIds.some((id) => id.equals(config._id))) ?? + []; + if (states.length == 0) { throw new ReplicationAssertionError( - `Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}` + `Failed to update checkpoint - no matching sync_config for _id: ${this.replicationStreamId}/${this.syncConfigIds + .map((id) => id.toHexString()) + .join(',')}` ); } + // The replication job / MongoBucketBatch must be constructed with all replicating (PROCESSING / ACTIVE) + // sync configs in the stream, otherwise we'll get inconsistencies. Configs in other states (e.g. STOP) + // remain embedded in the document and are ignored here. + const missingSyncConfig = preUpdateDocument!.sync_configs.find( + (config) => + [storage.SyncRuleState.PROCESSING, storage.SyncRuleState.ACTIVE].includes(config.state) && + !this.syncConfigIds.some((id) => id.equals(config._id)) + ); + if (missingSyncConfig != null) { + throw new ReplicationAssertionError(`Replication job not configured for sync config ${missingSyncConfig._id}`); + } - const canCheckpoint = canCheckpointState(lsn, { - snapshotDone: state.snapshot_done === true, - lastCheckpointLsn: state.last_checkpoint_lsn, - noCheckpointBefore: state.no_checkpoint_before - }); + // Effective head of the stream's op sequence. + // A combined flush supplies its new head; an empty commit uses the fenced persisted head. + const newCheckpoint = lastOp ?? this.persistedOpHead(preUpdateDocument); - if (!canCheckpoint) { - checkpointBlocked = true; - // Log the first blocked config's state. - checkpointLogState ??= { - snapshot_done: state.snapshot_done, - last_checkpoint_lsn: state.last_checkpoint_lsn, - no_checkpoint_before: state.no_checkpoint_before - }; - continue; - } + let checkpointBlocked = false; + let checkpointCreated = false; + let checkpointLogState: unknown = null; + const unblockedConfigIds: bson.ObjectId[] = []; - checkpointCreated ||= createEmptyCheckpoints || state.last_checkpoint !== newCheckpoint; - unblockedConfigIds.push(state._id); - } + for (const state of states) { + if (state.last_checkpoint != null && state.last_checkpoint > newCheckpoint) { + // last_persisted_op is $max-advanced durably in the same transaction as every flush, and + // checkpoints are only ever created at that head, so a checkpoint past the head means the + // op sequence or the stored state is corrupt. + throw new ReplicationAssertionError( + `Invariant violation: sync config ${state._id} has last_checkpoint ${state.last_checkpoint} > stream head ${newCheckpoint}` + ); + } - // Every commit advances the stream's resume position: commit() flushes first, so all - // source changes up to this lsn have been persisted, even when checkpoints are blocked. - // In the future we could also advance this on flush, when the connector provides the - // current position (see setResumeLsn, which connectors may already call after flushing). - const resumeLsnUpdate = { resume_lsn: lsn }; - - if (unblockedConfigIds.length > 0) { - // All unblocked configs get the SAME new value, so we apply it with a single updateOne - // (single-document atomicity). - const updateSet: Record = { - last_keepalive_ts: now, - last_fatal_error: null, - last_fatal_error_ts: null - }; - // Only advance checkpoint fields when an actual (non-empty) checkpoint is created, matching - // the previous per-config / v1 behaviour. - if (checkpointCreated) { - updateSet['sync_configs.$[config].last_checkpoint'] = newCheckpoint; - updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn; - updateSet['last_checkpoint_ts'] = now; + const canCheckpoint = canCheckpointState(lsn, { + snapshotDone: state.snapshot_done === true, + lastCheckpointLsn: state.last_checkpoint_lsn, + noCheckpointBefore: state.no_checkpoint_before + }); + + if (!canCheckpoint) { + checkpointBlocked = true; + // Log the first blocked config's state. + checkpointLogState ??= { + snapshot_done: state.snapshot_done, + last_checkpoint_lsn: state.last_checkpoint_lsn, + no_checkpoint_before: state.no_checkpoint_before + }; + continue; + } + + checkpointCreated ||= createEmptyCheckpoints || state.last_checkpoint !== newCheckpoint; + unblockedConfigIds.push(state._id); } - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId, - 'sync_configs._id': { $in: unblockedConfigIds } - }, - { $set: updateSet, $max: resumeLsnUpdate }, - { - session: this.session, - arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined + // Every commit advances the stream's resume position: commit() flushes first, so all + // source changes up to this lsn have been persisted, even when checkpoints are blocked. + // In the future we could also advance this on flush, when the connector provides the + // current position (see setResumeLsn, which connectors may already call after flushing). + const resumeLsnUpdate = { + resume_lsn: lsn, + ...(lastOp == null ? {} : { last_persisted_op: lastOp }) + }; + + if (unblockedConfigIds.length > 0) { + // All unblocked configs get the SAME new value, so we apply it with a single updateOne + // (single-document atomicity). + const updateSet: Record = { + last_fatal_error: null, + last_fatal_error_ts: null + }; + // Only advance checkpoint fields when an actual (non-empty) checkpoint is created, matching + // the previous per-config / v1 behaviour. + if (checkpointCreated) { + updateSet['sync_configs.$[config].last_checkpoint'] = newCheckpoint; + updateSet['sync_configs.$[config].last_checkpoint_lsn'] = lsn; + updateSet['last_checkpoint_ts'] = now; } - ); - } else { - // All selected configs are blocked - only update keepalive/error tracking and the - // resume position. - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId - }, - { - $set: { - last_keepalive_ts: now, - last_fatal_error: null, - last_fatal_error_ts: null + + writes.updateOne( + this.db.sync_rules, + { + _id: this.replicationStreamId, + 'sync_configs._id': { $in: unblockedConfigIds } }, - $max: resumeLsnUpdate - }, - { session: this.session } - ); - } + { $set: updateSet, $max: resumeLsnUpdate }, + { + arrayFilters: checkpointCreated ? [{ 'config._id': { $in: unblockedConfigIds } }] : undefined + } + ); + } else { + // All selected configs are blocked - only update keepalive/error tracking and the + // resume position. + writes.updateOne( + this.db.sync_rules, + { + _id: this.replicationStreamId + }, + { + $set: { + last_fatal_error: null, + last_fatal_error_ts: null + }, + $max: resumeLsnUpdate + } + ); + } + await writes.execute(); + return { checkpointBlocked, checkpointCreated, checkpointLogState, newCheckpoint }; + }; + const projection = { sync_configs: 1, last_persisted_op: 1 }; + const { checkpointBlocked, checkpointCreated, checkpointLogState, newCheckpoint } = await this.flushAndCommit( + checkpoint, + () => this.withFencedTransaction(() => this.fence(this.session, projection), checkpoint), + options, + projection + ); if (checkpointBlocked) { if (Date.now() - this.lastWaitingLogThrottledV3 > 5_000) { this.logger.info( @@ -446,23 +438,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { async setResumeLsn(lsn: string): Promise { using _ = this.tracer.span('storage', 'set_resume_lsn'); - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId - }, - { - $set: { - resume_lsn: lsn - } - }, - { - session: this.session, - // Losing occasional resume LSN is fine. That may mean reprocessing - // some source changes in some edge cases, which is not an issue since - // changes are processed in an idempotent way. - writeConcern: { w: 1 } - } - ); + // Losing occasional resume LSN would only reprocess source changes. + // Keep the lease check atomic, but do not wait for majority replication of this resume hint. + await this.updateStreamMetadata({ resume_lsn: { $literal: lsn } }, {}, { w: 1 }); } private async autoActivateV3(lsn: string): Promise { @@ -473,7 +451,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { const session = this.session; let activated = false; let needsFutureActivationCheck = true; - await session.withTransaction(async () => { + await this.withTransaction(async () => { // Reset on transaction retries. needsFutureActivationCheck = true; activated = false; @@ -561,7 +539,7 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { } else if (doc.state == storage.SyncRuleState.ACTIVE && processingStates.length == 0) { needsFutureActivationCheck = false; } - }); + }, session); if (activated) { this.logger.info(`Activated new replication stream at ${lsn}`); await this.db.notifyCheckpoint(); @@ -571,26 +549,35 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { } } - async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise { - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId, - 'sync_configs._id': { $in: this.syncConfigIds } - }, - { - $set: { - 'sync_configs.$[config].snapshot_done': true, - last_keepalive_ts: new Date() - }, - $max: { - 'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn + private syncConfigMetadataUpdate(syncConfigIds: bson.ObjectId[], set: lib_mongo.mongo.Document) { + return { + sync_configs: { + $map: { + input: '$sync_configs', + as: 'config', + in: { + $cond: [ + { $in: ['$$config._id', { $literal: syncConfigIds }] }, + { $mergeObjects: ['$$config', set] }, + '$$config' + ] + } } - }, - { - session: this.session, - arrayFilters: [{ 'config._id': { $in: this.syncConfigIds } }] } - ); + }; + } + + private snapshotDoneUpdate(no_checkpoint_before_lsn: string) { + return this.syncConfigMetadataUpdate(this.syncConfigIds, { + snapshot_done: true, + no_checkpoint_before: { $max: ['$$config.no_checkpoint_before', { $literal: no_checkpoint_before_lsn }] } + }); + } + + async markAllSnapshotDone(no_checkpoint_before_lsn: string): Promise { + await this.updateStreamMetadata(this.snapshotDoneUpdate(no_checkpoint_before_lsn), { + 'sync_configs._id': { $in: this.syncConfigIds } + }); } async markSnapshotDone(no_checkpoint_before_lsn: string, options?: { throwOnConflict?: boolean }): Promise { @@ -614,7 +601,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { } } - await this.markAllSnapshotDone(no_checkpoint_before_lsn); + await this.updateStreamMetadataInTransaction(this.snapshotDoneUpdate(no_checkpoint_before_lsn), { + 'sync_configs._id': { $in: this.syncConfigIds } + }); }); } @@ -623,22 +612,9 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { if (syncConfigIds.length == 0) { return; } - - await this.db.sync_rules.updateOne( - { - _id: this.replicationStreamId, - 'sync_configs._id': { $in: syncConfigIds } - }, - { - $set: { - 'sync_configs.$[config].snapshot_done': false - } - }, - { - session: this.session, - arrayFilters: [{ 'config._id': { $in: syncConfigIds } }] - } - ); + await this.updateStreamMetadata(this.syncConfigMetadataUpdate(syncConfigIds, { snapshot_done: false }), { + 'sync_configs._id': { $in: syncConfigIds } + }); } async markTableSnapshotDone( @@ -672,9 +648,6 @@ export class MongoBucketBatchV3 extends MongoBucketBatch { 'sync_configs._id': { $in: syncConfigIds } }, { - $set: { - last_keepalive_ts: new Date() - }, $max: { 'sync_configs.$[config].no_checkpoint_before': no_checkpoint_before_lsn } diff --git a/modules/module-mongodb-storage/src/utils/test-utils.ts b/modules/module-mongodb-storage/src/utils/test-utils.ts index 861bd6cfc..11a082463 100644 --- a/modules/module-mongodb-storage/src/utils/test-utils.ts +++ b/modules/module-mongodb-storage/src/utils/test-utils.ts @@ -7,12 +7,13 @@ import { PowerSyncMongo } from '../storage/implementation/db.js'; export type MongoTestStorageOptions = { url: string; isCI: boolean; + monitorCommands?: boolean; } & Omit; export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { return { factory: async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.monitorCommands); // None of the tests insert data into this collection, so it was never created if (!(await db.db.listCollections({ name: db.bucket_parameters.collectionName }).hasNext())) { @@ -41,7 +42,7 @@ export function mongoTestStorageFactoryGenerator(factoryOptions: MongoTestStorag export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTestStorageOptions) { return async (options?: TestStorageOptions) => { - const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI); + const db = connectMongoForTests(factoryOptions.url, factoryOptions.isCI, factoryOptions.monitorCommands); await db.createConnectionReportingCollection(); @@ -53,10 +54,11 @@ export function mongoTestReportStorageFactoryGenerator(factoryOptions: MongoTest }; } -export const connectMongoForTests = (url: string, isCI: boolean) => { +export const connectMongoForTests = (url: string, isCI: boolean, monitorCommands = false) => { // Short timeout for tests, to fail fast when the server is not available. // Slightly longer timeouts for CI, to avoid arbitrary test failures const client = new mongo.MongoClient(url, { + monitorCommands, connectTimeoutMS: isCI ? 15_000 : 5_000, socketTimeoutMS: isCI ? 15_000 : 5_000, serverSelectionTimeoutMS: isCI ? 15_000 : 2_500 diff --git a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap index e514d98dd..fc21a410f 100644 --- a/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap +++ b/modules/module-mongodb-storage/test/src/__snapshots__/storage_sync.test.ts.snap @@ -270,7 +270,7 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for stre ], }, ], - "last_op_id": "2", + "last_op_id": "65537", "streams": [ { "errors": [], @@ -292,17 +292,17 @@ exports[`sync - mongodb > storage v1 > encodes sync rules id in buckets for stre "object_id": "t1", "object_type": "test", "op": "PUT", - "op_id": "2", + "op_id": "65537", "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, - "next_after": "2", + "next_after": "65537", }, }, { "checkpoint_complete": { - "last_op_id": "2", + "last_op_id": "65537", }, }, ] @@ -1265,7 +1265,7 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for stre ], }, ], - "last_op_id": "2", + "last_op_id": "65537", "streams": [ { "errors": [], @@ -1287,17 +1287,17 @@ exports[`sync - mongodb > storage v2 > encodes sync rules id in buckets for stre "object_id": "t1", "object_type": "test", "op": "PUT", - "op_id": "2", + "op_id": "65537", "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, - "next_after": "2", + "next_after": "65537", }, }, { "checkpoint_complete": { - "last_op_id": "2", + "last_op_id": "65537", }, }, ] @@ -2849,7 +2849,7 @@ exports[`sync - mongodb > storage v4 > encodes sync rules id in buckets for stre ], }, ], - "last_op_id": "2", + "last_op_id": "65537", "streams": [ { "errors": [], @@ -2871,17 +2871,17 @@ exports[`sync - mongodb > storage v4 > encodes sync rules id in buckets for stre "object_id": "t1", "object_type": "test", "op": "PUT", - "op_id": "2", + "op_id": "65537", "subkey": "bfe6a7fc-1a36-5a95-877f-518ff63ecb56", }, ], "has_more": false, - "next_after": "2", + "next_after": "65537", }, }, { "checkpoint_complete": { - "last_op_id": "2", + "last_op_id": "65537", }, }, ] diff --git a/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts b/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts index 02566f012..415dd5eb6 100644 --- a/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts +++ b/modules/module-mongodb-storage/test/src/cleanup-stopped-sync-configs.test.ts @@ -2,7 +2,7 @@ import { MongoSyncBucketStorageV3 } from '@module/storage/implementation/v3/Mong import { ObjectStorageLifecycle } from '@module/storage/implementation/v3/object-storage/ObjectStorageLifecycle.js'; import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { getTestStorage, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -78,7 +78,7 @@ streams: { storageVersion: 3 } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3; const db = firstStorage.db as VersionedPowerSyncMongoV3; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -155,7 +155,7 @@ streams: ); expect(second.replicationStreamId).toBe(first.replicationStreamId); const replicatingStreams = await factory.getReplicatingReplicationStreams(); - const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; + const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); await secondWriter.markAllSnapshotDone('2/1'); await secondWriter.commit('2/1'); @@ -218,7 +218,7 @@ streams: { storageVersion: 3 } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3; const db = firstStorage.db as VersionedPowerSyncMongoV3; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await firstWriter.resolveTables({ @@ -266,7 +266,7 @@ streams: expect(secondDefinitionId).toBe(projectDefinitionId); const replicatingStreams = await factory.getReplicatingReplicationStreams(); - const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; + const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); await secondWriter.markAllSnapshotDone('2/1'); await secondWriter.commit('2/1'); @@ -289,7 +289,10 @@ streams: expect(await collectionExists(db, ownerBucketDataCollection)).toBe(false); expect(await collectionExists(db, sourceRecordsCollection)).toBe(true); - const activeStorage = (await factory.getActiveSyncConfig())!.storage as MongoSyncBucketStorageV3; + const activeStorage = (await getTestStorage( + factory, + (await factory.getActiveSyncConfig())!.replicationStream + )) as MongoSyncBucketStorageV3; await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS); const activeTable = ( await activeWriter.resolveTables({ @@ -343,7 +346,7 @@ event_definitions: { storageVersion: 3 } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3; const db = firstStorage.db as VersionedPowerSyncMongoV3; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await firstWriter.resolveTables({ @@ -402,7 +405,7 @@ event_definitions: expect(second.replicationStreamId).toBe(first.replicationStreamId); const replicatingStreams = await factory.getReplicatingReplicationStreams(); - const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; + const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); await secondWriter.markAllSnapshotDone('2/1'); await secondWriter.commit('2/1'); @@ -466,7 +469,7 @@ event_definitions: { storageVersion: 3 } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3; const db = firstStorage.db as VersionedPowerSyncMongoV3; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -538,7 +541,7 @@ streams: expect(second.replicationStreamId).toBe(first.replicationStreamId); const replicatingStreams = await factory.getReplicatingReplicationStreams(); - const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; + const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); await secondWriter.markAllSnapshotDone('2/1'); await secondWriter.commit('2/1'); @@ -599,7 +602,7 @@ streams: { storageVersion: 3 } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorageV3; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorageV3; const db = firstStorage.db as VersionedPowerSyncMongoV3; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await firstWriter.resolveTables({ @@ -652,7 +655,7 @@ streams: expect(secondIndexId).toBe(roleIndexId); const replicatingStreams = await factory.getReplicatingReplicationStreams(); - const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorageV3; + const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorageV3; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); await secondWriter.markAllSnapshotDone('2/1'); await secondWriter.commit('2/1'); @@ -664,7 +667,10 @@ streams: expect(await collectionExists(db, orgParameterIndexCollection)).toBe(false); expect(await collectionExists(db, sourceRecordsCollection)).toBe(true); - const activeStorage = (await factory.getActiveSyncConfig())!.storage as MongoSyncBucketStorageV3; + const activeStorage = (await getTestStorage( + factory, + (await factory.getActiveSyncConfig())!.replicationStream + )) as MongoSyncBucketStorageV3; await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS); const activeTable = ( await activeWriter.resolveTables({ diff --git a/modules/module-mongodb-storage/test/src/concurrent_writers.test.ts b/modules/module-mongodb-storage/test/src/concurrent_writers.test.ts new file mode 100644 index 000000000..cff9cdd67 --- /dev/null +++ b/modules/module-mongodb-storage/test/src/concurrent_writers.test.ts @@ -0,0 +1,599 @@ +import { mongo } from '@powersync/lib-service-mongodb'; +import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; +import { execFile } from 'node:child_process'; +import { fileURLToPath } from 'node:url'; +import { promisify } from 'node:util'; +import { describe, expect, test, vi } from 'vitest'; +import { PersistedBatch } from '../../src/storage/implementation/common/PersistedBatch.js'; +import { MongoOpIdAllocator } from '../../src/storage/implementation/MongoOpIdAllocator.js'; +import { MongoPersistedReplicationStream } from '../../src/storage/implementation/MongoPersistedReplicationStream.js'; +import { mongoTestStorageFactoryGenerator } from '../../src/utils/test-utils.js'; +import { env } from './env.js'; + +const factoryGen = mongoTestStorageFactoryGenerator({ url: env.MONGO_TEST_URL, isCI: env.CI }); +const rules = `bucket_definitions: + global: + data: + - SELECT id, description FROM items +`; + +async function openStream(factory: Awaited>, version: number) { + const stream = await factory.updateSyncRules(updateSyncRulesFromYaml(rules, { storageVersion: version })); + const bucketStorage = await test_utils.getTestStorage(factory, stream); + const writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, stream.replicationStreamId); + await writer.markAllSnapshotDone('1/1'); + return { stream, bucketStorage, writer, table }; +} + +async function insert(writer: storage.BucketStorageBatch, table: storage.SourceTable, id: string) { + await writer.save({ + sourceTable: table, + tag: storage.SaveOperationTag.INSERT, + after: { id, description: id }, + afterReplicaId: test_utils.rid(id) + }); +} + +describe.each([1, 2, 4])('concurrent writers v%s', (version) => { + test('requires a lease to create a writer, including after another lease is released', async () => { + await using factory = await factoryGen.factory(); + const stream = await factory.updateSyncRules(updateSyncRulesFromYaml(rules, { storageVersion: version })); + const unleased = factory.getInstance(stream); + await expect(unleased.createWriter(test_utils.BATCH_OPTIONS)).rejects.toThrow('replication lease is required'); + const lock = await stream.lock(); + try { + const leased = factory.getInstance(stream, { replicationLock: lock }); + await using writer = await leased.createWriter(test_utils.BATCH_OPTIONS); + await lock.release(); + await expect(writer.commit('1/2')).rejects.toThrow('Replication lock released'); + await expect(leased.createWriter(test_utils.BATCH_OPTIONS)).rejects.toThrow('Replication lock released'); + await expect(unleased.createWriter(test_utils.BATCH_OPTIONS)).rejects.toThrow('replication lease is required'); + } finally { + await lock.release(); + } + }); + + test('combines the final flush and checkpoint into one transaction', async () => { + const monitored = mongoTestStorageFactoryGenerator({ + url: env.MONGO_TEST_URL, + isCI: env.CI, + monitorCommands: true + }); + await using factory = await monitored.factory(); + const a = await openStream(factory, version); + await using writer = a.writer; + // Warm up activation, the reservation and error clearing before measuring. + await insert(writer, a.table, 'warmup'); + await writer.commit('1/2'); + const commands: string[] = []; + const listener = (event: mongo.CommandStartedEvent) => { + if (event.command.autocommit === false) { + commands.push(event.commandName); + } + }; + factory.db.client.on('commandStarted', listener); + try { + await insert(writer, a.table, 'separate'); + await writer.flush(); + await writer.commit('1/3'); + const separate = [...commands]; + commands.length = 0; + await insert(writer, a.table, 'combined'); + await writer.commit('1/4'); + expect(separate.filter((name) => name === 'commitTransaction')).toHaveLength(2); + expect(commands.filter((name) => name === 'commitTransaction')).toHaveLength(1); + expect(separate.length - commands.length).toBe(version < 3 ? 2 : 3); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(3n); + } finally { + factory.db.client.off('commandStarted', listener); + } + }); + + test('only checkpoints the last piece of a split flush', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writer = a.writer; + await insert(writer, a.table, 'warmup'); + await writer.commit('1/2'); + // Force a split after each row without needing a transaction-sized fixture. + const split = vi.spyOn(PersistedBatch.prototype, 'shouldFlushTransaction').mockReturnValue(true); + const flush = PersistedBatch.prototype.flush; + const observed: bigint[] = []; + const inspect = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementation(async function ( + this: PersistedBatch, + ...args + ) { + observed.push((await a.bucketStorage.getCheckpoint()).checkpoint); + return flush.apply(this, args); + }); + try { + await insert(writer, a.table, 'first'); + await insert(writer, a.table, 'last'); + await writer.commit('1/3'); + expect(observed).toEqual([1n, 1n]); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(3n); + } finally { + inspect.mockRestore(); + split.mockRestore(); + } + }); + + test('retries the final rows and checkpoint together after a transaction conflict', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writer = a.writer; + await insert(writer, a.table, 'warmup'); + await writer.commit('1/2'); + const withTransaction = mongo.ClientSession.prototype.withTransaction; + let attempts = 0; + const retry = vi.spyOn(mongo.ClientSession.prototype, 'withTransaction').mockImplementationOnce(function ( + this: mongo.ClientSession, + callback, + options + ) { + return withTransaction.call( + this, + async (session) => { + const result = await callback(session); + // Neither the final rows nor their checkpoint have committed yet. + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(1n); + expect(writer.last_flushed_op).toBe(1n); + attempts += 1; + if (attempts === 1) { + throw new mongo.MongoServerError({ + message: 'retry checkpoint', + code: 112, + errorLabels: ['TransientTransactionError'] + }); + } + return result; + }, + options + ); + }); + try { + await insert(writer, a.table, 'retried'); + await writer.commit('1/3'); + expect(attempts).toBe(2); + expect(writer.last_flushed_op).toBe(2n); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(2n); + } finally { + retry.mockRestore(); + } + }); + + test('a blocked combined checkpoint preserves the head for a new writer', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writer = a.writer; + await writer.markAllSnapshotDone('1/9'); + await insert(writer, a.table, 'blocked'); + expect(await writer.commit('1/2')).toMatchObject({ checkpointBlocked: true, checkpointCreated: false }); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(0n); + await using resumed = await a.bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await resumed.commit('1/9'); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(writer.last_flushed_op); + expect(writer.last_flushed_op).toBe(1n); + }); + + test('same-stream writers queue FIFO before starting transactions', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using first = a.writer; + await insert(first, a.table, 'warmup'); + await first.commit('1/2'); + // Separate storage instances must still share the lease's queue. + const otherStorage = factory.getInstance(a.stream); + await using checkpointWriter = await otherStorage.createWriter(test_utils.BATCH_OPTIONS); + await using last = await otherStorage.createWriter(test_utils.BATCH_OPTIONS); + const lastTable = await test_utils.resolveTestTable( + last, + 'items', + ['id'], + factoryGen, + a.stream.replicationStreamId + ); + await insert(first, a.table, 'first'); + await insert(last, lastTable, 'last'); + + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const queued = Promise.withResolvers(); + const mutex = (a.stream as MongoPersistedReplicationStream).current_lock!.writerMutex; + const exclusiveLock = mutex.exclusiveLock; + let admissions = 0; + const admission = vi.spyOn(mutex, 'exclusiveLock').mockImplementation(function (callback) { + const result = exclusiveLock.call(mutex, callback); + if (++admissions === 3) queued.resolve(); + return result; + }); + const transactions = vi.spyOn(mongo.ClientSession.prototype, 'withTransaction'); + const flush = PersistedBatch.prototype.flush; + const stalled = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function ( + this: PersistedBatch, + ...args + ) { + entered.resolve(); + await release.promise; + return flush.apply(this, args); + }); + const pending: Promise[] = []; + try { + pending.push(first.flush()); + await entered.promise; + // The empty checkpoint must run before the later flush. + pending.push(checkpointWriter.commit('1/3')); + pending.push(last.flush()); + await queued.promise; + expect(transactions).toHaveBeenCalledTimes(1); + release.resolve(); + await Promise.all(pending); + expect(first.last_flushed_op).toBe(2n); + expect(last.last_flushed_op).toBe(3n); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(2n); + } finally { + release.resolve(); + await Promise.allSettled(pending); + stalled.mockRestore(); + transactions.mockRestore(); + admission.mockRestore(); + } + }); + + test('another stream publishes while a transaction is stalled, without touching its reserved IDs', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writerA = a.writer; + // An independent factory/client has no shared allocator or in-process coordination. + await using other = await factoryGen.factory({ doNotClear: true }); + const b = await openStream(other, version); + await using writerB = b.writer; + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const flush = PersistedBatch.prototype.flush; + const stalled = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function ( + this: PersistedBatch, + ...args + ) { + entered.resolve(); + await release.promise; + return flush.apply(this, args); + }); + await insert(writerA, a.table, 'a'); + const publishingA = writerA.flush(); + try { + await entered.promise; + await insert(writerB, b.table, 'b'); + await writerB.commit('1/2'); + expect((await b.bucketStorage.getCheckpoint()).checkpoint).toBeGreaterThan(1n); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(0n); + } finally { + release.resolve(); + await publishingA; + stalled.mockRestore(); + } + expect(writerA.last_flushed_op).toBe(1n); + // A can keep using its lower range: B's checkpoint belongs to another stream. + await insert(writerA, a.table, 'a2'); + await writerA.flush(); + expect(writerA.last_flushed_op).toBe(2n); + }); + + test('another process can reserve and checkpoint while this process holds a stream transaction', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writerA = a.writer; + const b = await openStream(factory, version); + await b.writer.dispose(); + await test_utils.releaseTestStorageLease(factory, b.stream.replicationStreamId); + const entered = Promise.withResolvers(); + const release = Promise.withResolvers(); + const flush = PersistedBatch.prototype.flush; + const stalled = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function ( + this: PersistedBatch, + ...args + ) { + entered.resolve(); + await release.promise; + return flush.apply(this, args); + }); + await insert(writerA, a.table, 'a'); + const pending = writerA.flush(); + try { + await entered.promise; + const child = await promisify(execFile)( + process.execPath, + [ + fileURLToPath(new URL('./helpers/concurrentWriter.mjs', import.meta.url)), + env.MONGO_TEST_URL, + `${b.stream.replicationStreamId}` + ], + { timeout: 10_000 } + ).catch((error) => { + throw new Error(`${error.message}\n${error.stdout}\n${error.stderr}`); + }); + expect(child.stdout).toMatch(/checkpoint=[1-9][0-9]+/); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(0n); + } finally { + release.resolve(); + await pending; + stalled.mockRestore(); + } + }, 15_000); + + test('reuses one reservation across writers and flushes and retries rolled-back writes', async () => { + await using factory = await factoryGen.factory(); + const { writer: batch, table, bucketStorage } = await openStream(factory, version); + await using writer = batch; + const reservations = vi.spyOn(MongoOpIdAllocator.prototype, 'reserve'); + const flush = PersistedBatch.prototype.flush; + const retry = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function ( + this: PersistedBatch, + ...args + ) { + await flush.apply(this, args); + throw new mongo.MongoServerError({ message: 'retry', code: 112, errorLabels: ['TransientTransactionError'] }); + }); + try { + for (let i = 0; i < 3; i++) { + await insert(writer, table, `${i}`); + await writer.flush(); + } + await writer.commit('1/2'); + expect(reservations).toHaveBeenCalledTimes(1); + expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(3n); + await using nextWriter = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const nextTable = await test_utils.resolveTestTable(nextWriter, 'items', ['id'], factoryGen, 1); + await insert(nextWriter, nextTable, 'next'); + await nextWriter.commit('1/3'); + expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(4n); + expect(reservations).toHaveBeenCalledTimes(1); + expect((await factory.db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBeGreaterThan(3n); + } finally { + reservations.mockRestore(); + retry.mockRestore(); + } + }); + + test.each(['during flush', 'after flush'])('graceful cancellation %s preserves snapshot progress', async (when) => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await a.writer.dispose(); + const abort = new AbortController(); + await using writer = await a.bucketStorage.createWriter({ ...test_utils.BATCH_OPTIONS, signal: abort.signal }); + const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, a.stream.replicationStreamId); + const flush = PersistedBatch.prototype.flush; + const interrupted = vi.spyOn(PersistedBatch.prototype, 'flush').mockImplementationOnce(async function ( + this: PersistedBatch, + ...args + ) { + const result = await flush.apply(this, args); + if (when === 'during flush') { + abort.abort(); + } + return result; + }); + try { + await insert(writer, table, 'page'); + await writer.flush(); + if (when === 'after flush') { + abort.abort(); + } + await writer.updateTableProgress(table, { replicatedCount: 1, totalEstimatedCount: 1 }); + const persisted = await writer.getSourceTableStatus(table); + expect(persisted?.snapshotStatus?.replicatedCount).toBe(1); + await writer.commit('1/2'); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(1n); + } finally { + interrupted.mockRestore(); + } + }); + + test('recovers flushed operations through existing version-specific fields', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writer = a.writer; + await insert(writer, a.table, 'checkpointed'); + await writer.commit('1/2'); + await insert(writer, a.table, 'flushed'); + await writer.flush(); + const document = await factory.db.sync_rules.findOne({ _id: a.stream.replicationStreamId }); + expect(document).not.toHaveProperty('writer_transaction'); + if (version < 3) { + expect(document).not.toHaveProperty('last_persisted_op'); + expect(document).toMatchObject({ last_checkpoint: 1n, keepalive_op: '2' }); + } else { + expect(document).toHaveProperty('last_persisted_op', 2n); + } + await using other = await factoryGen.factory({ doNotClear: true }); + const stream = (await other.getReplicatingReplicationStreams())[0]; + await test_utils.releaseTestStorageLease(factory, a.stream.replicationStreamId); + const bucketStorage = await test_utils.getTestStorage(other, stream); + await using resumed = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + await resumed.commit('1/3'); + expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(2n); + if (version < 3) { + expect(await factory.db.sync_rules.findOne({ _id: stream.replicationStreamId })).toMatchObject({ + last_checkpoint: 2n, + keepalive_op: null + }); + } + }); + + test('fencing changes the heartbeat even when it is ahead of the clock', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writer = a.writer; + const future = new Date(Date.now() + 60_000); + await factory.db.sync_rules.updateOne( + { _id: a.stream.replicationStreamId }, + { $set: { last_keepalive_ts: future } } + ); + await writer.setResumeLsn('1/2'); + const document = await factory.db.sync_rules.findOne({ _id: a.stream.replicationStreamId }); + expect(document!.last_keepalive_ts!.getTime()).toBe(future.getTime() + 1); + expect(document).not.toHaveProperty('writer_transaction'); + }); + + test('independent writers of the same stream never publish behind its durable head', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, version); + await using writerA = a.writer; + await insert(writerA, a.table, 'a'); + await writerA.flush(); + await using other = await factoryGen.factory({ doNotClear: true }); + const reloaded = (await other.getReplicatingReplicationStreams()).find( + (s) => s.replicationStreamId === a.stream.replicationStreamId + )!; + await using writerB = await other + .getInstance(reloaded, { replicationLock: (a.stream as MongoPersistedReplicationStream).current_lock! }) + .createWriter(test_utils.BATCH_OPTIONS); + const tableB = await test_utils.resolveTestTable(writerB, 'items', ['id'], factoryGen, 1); + await insert(writerB, tableB, 'b'); + await writerB.flush(); + const higher = writerB.last_flushed_op!; + await insert(writerA, a.table, 'c'); + await writerA.flush(); + expect(writerA.last_flushed_op).toBeGreaterThan(higher); + await writerA.commit('1/2'); + expect((await a.bucketStorage.getCheckpoint()).checkpoint).toBe(writerA.last_flushed_op); + }); + + test('lease takeover fences stale row, metadata, and checkpoint writes', async () => { + await using factory = await factoryGen.factory(); + const initial = await openStream(factory, version); + await initial.writer.dispose(); + const stream = initial.stream as MongoPersistedReplicationStream; + await test_utils.releaseTestStorageLease(factory, stream.replicationStreamId); + const first = await stream.lock(); + await using firstLifetime = { [Symbol.asyncDispose]: () => first.release() }; + await using oldWriter = await factory.getInstance(stream).createWriter(test_utils.BATCH_OPTIONS); + await using oldCheckpointWriter = await factory.getInstance(stream).createWriter(test_utils.BATCH_OPTIONS); + const oldTable = await test_utils.resolveTestTable(oldWriter, 'items', ['id'], factoryGen, 1); + await insert(oldWriter, oldTable, 'old'); + await oldWriter.flush(); + const abandonedEnd = (await factory.db.op_id_sequence.findOne({ _id: 'main' }))!.op_id; + await factory.db.sync_rules.updateOne( + { _id: stream.replicationStreamId }, + { $set: { 'lock.expires_at': new Date(0) } } + ); + await using other = await factoryGen.factory({ doNotClear: true }); + const nextStream = (await other.getReplicatingReplicationStreams())[0]; + const next = await nextStream.lock(); + await using nextLifetime = { [Symbol.asyncDispose]: () => next.release() }; + await using nextWriter = await other.getInstance(nextStream).createWriter(test_utils.BATCH_OPTIONS); + const nextTable = await test_utils.resolveTestTable(nextWriter, 'items', ['id'], factoryGen, 1); + await insert(nextWriter, nextTable, 'new'); + await nextWriter.commit('1/2'); + expect(nextWriter.last_flushed_op).toBeGreaterThan(abandonedEnd); + await insert(oldWriter, oldTable, 'stale'); + await expect(oldWriter.flush()).rejects.toThrow('no longer owns'); + await expect(oldWriter.setResumeLsn('9/9')).rejects.toThrow('no longer owns'); + await expect(oldWriter.markAllSnapshotDone('9/9')).rejects.toThrow('no longer owns'); + await expect(oldWriter.markTableSnapshotRequired(oldTable)).rejects.toThrow('no longer owns'); + await expect(oldWriter.commit('9/9')).rejects.toThrow('no longer owns'); + // No pending rows: this exercises the checkpoint fence, not the flush fence. + await expect(oldCheckpointWriter.commit('9/9')).rejects.toThrow('no longer owns'); + await first.release(); + expect((await factory.db.sync_rules.findOne({ _id: stream.replicationStreamId }))!.lock?.id).toBe( + (next as typeof first).lock_id + ); + }); + + test.each([false, true])('extends a nearly exhausted range (fallback: %s)', async (fallback) => { + await using factory = await factoryGen.factory(); + const { writer: batch, table, bucketStorage } = await openStream(factory, version); + await using writer = batch; + const allocator = factory.getOpIdAllocator( + bucketStorage.replicationStream, + bucketStorage.replicationStream.current_lock! + ); + await allocator.reserve(); + allocator.committed(65_535n); + // Simulate capacity becoming insufficient after the pre-transaction check. + // The fallback must still abort and retry safely in that case. + const capacity = vi.spyOn(allocator, 'ensureCapacity'); + if (fallback) { + capacity.mockResolvedValueOnce(undefined); + } + const sequences = vi.spyOn(allocator, 'sequence'); + try { + await insert(writer, table, 'first'); + await insert(writer, table, 'second'); + await writer.commit('1/2'); + expect(writer.last_flushed_op).toBe(65_537n); + expect((await bucketStorage.getCheckpoint()).checkpoint).toBe(65_537n); + expect(sequences).toHaveBeenCalledTimes(fallback ? 2 : 1); + } finally { + capacity.mockRestore(); + sequences.mockRestore(); + } + }); +}); + +describe('operation ID reservations', () => { + test('initializes and reserves disjoint ranges concurrently in one command each', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, 4); + await using writer = a.writer; + const db = factory.db.versioned(a.bucketStorage.replicationStream.getStorageConfig()); + const allocators = Array.from({ length: 4 }, () => new MongoOpIdAllocator(db)); + const initialize = vi.spyOn(db.op_id_sequence, 'updateOne'); + const reserve = vi.spyOn(db.op_id_sequence, 'findOneAndUpdate'); + try { + await Promise.all(allocators.map((allocator) => allocator.reserve())); + const firstIds = allocators.map((allocator) => allocator.sequence(0n).next()).sort((a, b) => Number(a - b)); + expect(firstIds).toEqual([1n, 65_537n, 131_073n, 196_609n]); + expect(initialize).not.toHaveBeenCalled(); + expect(reserve).toHaveBeenCalledTimes(4); + } finally { + initialize.mockRestore(); + reserve.mockRestore(); + } + }); + + test('refills below 16k remaining IDs, preserving the tail across disjoint ranges', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, 4); + await using writer = a.writer; + const db = factory.db.versioned(a.bucketStorage.replicationStream.getStorageConfig()); + const allocator = factory.getOpIdAllocator( + a.bucketStorage.replicationStream, + a.bucketStorage.replicationStream.current_lock! + ); + await allocator.ensureCapacity(); + allocator.committed(49_152n); + await allocator.ensureCapacity(); + expect((await db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(65_536n); + + // Another stream's reservation creates a gap that must not count as capacity. + await new MongoOpIdAllocator(db).reserve(); + allocator.committed(49_153n); + await Promise.all([allocator.ensureCapacity(), allocator.ensureCapacity()]); + expect((await db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(196_608n); + expect(allocator.sequence(49_153n).next()).toBe(49_154n); + const sequence = allocator.sequence(65_535n); + expect(sequence.next()).toBe(65_536n); + expect(sequence.next()).toBe(131_073n); + await allocator.ensureCapacity(); + expect((await db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(196_608n); + }); + + test('refuses a reservation that would overflow without changing the watermark', async () => { + await using factory = await factoryGen.factory(); + const a = await openStream(factory, 4); + await using writer = a.writer; + const max = (1n << 63n) - 1n; + await factory.db.op_id_sequence.insertOne({ _id: 'main', op_id: max - 65_536n }); + const allocator = factory.getOpIdAllocator( + a.bucketStorage.replicationStream, + a.bucketStorage.replicationStream.current_lock! + ); + await allocator.reserve(); + expect(allocator.sequence(max - 1n).next()).toBe(max); + await expect(allocator.reserve()).rejects.toThrow('Operation ID sequence exhausted'); + expect((await factory.db.op_id_sequence.findOne({ _id: 'main' }))!.op_id).toBe(max); + }); +}); diff --git a/modules/module-mongodb-storage/test/src/helpers/concurrentWriter.mjs b/modules/module-mongodb-storage/test/src/helpers/concurrentWriter.mjs new file mode 100644 index 000000000..b0580deca --- /dev/null +++ b/modules/module-mongodb-storage/test/src/helpers/concurrentWriter.mjs @@ -0,0 +1,29 @@ +import { storage } from '@powersync/service-core'; +import { test_utils } from '@powersync/service-core-tests'; +import { mongoTestStorageFactoryGenerator } from '../../../dist/utils/test-utils.js'; + +// A separate process exercises the database protocol without sharing any mutex, +// allocator, MongoClient, or lease object with the test runner. +async function main() { + const factoryGen = mongoTestStorageFactoryGenerator({ url: process.argv[2], isCI: true }); + await using factory = await factoryGen.factory({ doNotClear: true }); + const streamId = Number(process.argv[3]); + const stream = (await factory.getReplicatingReplicationStreams()).find((s) => s.replicationStreamId === streamId); + const lock = await stream.lock(); + await using lockLifetime = { [Symbol.asyncDispose]: () => lock.release() }; + const bucketStorage = factory.getInstance(stream, { replicationLock: lock }); + await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); + const table = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, streamId); + await writer.save({ + sourceTable: table, + tag: storage.SaveOperationTag.INSERT, + after: { id: 'child', description: 'child' }, + afterReplicaId: test_utils.rid('child') + }); + await writer.commit('1/2'); + console.log(`checkpoint=${(await bucketStorage.getCheckpoint()).checkpoint}`); +} + +await main(); +// Workspace test utilities start background timers. Exit after all resources are disposed. +process.exit(0); diff --git a/modules/module-mongodb-storage/test/src/object_storage_usage.test.ts b/modules/module-mongodb-storage/test/src/object_storage_usage.test.ts index d0f628493..da74a6fd0 100644 --- a/modules/module-mongodb-storage/test/src/object_storage_usage.test.ts +++ b/modules/module-mongodb-storage/test/src/object_storage_usage.test.ts @@ -8,6 +8,7 @@ import { VersionedPowerSyncMongoV3 } from '@module/storage/implementation/v3/Ver import * as lib_mongo from '@powersync/lib-service-mongodb'; import { mongo } from '@powersync/lib-service-mongodb'; import { updateSyncRulesFromYaml } from '@powersync/service-core'; +import { getTestStorage } from '@powersync/service-core-tests'; import { BucketDefinitionId } from '@powersync/service-sync-rules'; import { describe, expect, test } from 'vitest'; import { INITIALIZED_MONGO_STORAGE_FACTORY } from './util.js'; @@ -33,7 +34,7 @@ type UsageEntry = { async function withUsageContext(callback: (context: UsageContext) => Promise): Promise { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const definitionId = syncRules.syncConfigContent[0].mapping.allBucketDefinitionIds()[0]; return callback({ db, bucketStorage, definitionId }); diff --git a/modules/module-mongodb-storage/test/src/parameter_compacting_v1.test.ts b/modules/module-mongodb-storage/test/src/parameter_compacting_v1.test.ts index e2e9c5da3..59383a472 100644 --- a/modules/module-mongodb-storage/test/src/parameter_compacting_v1.test.ts +++ b/modules/module-mongodb-storage/test/src/parameter_compacting_v1.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { getTestStorage, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import type { SyncRuleDocumentV1 } from '../../src/storage/implementation/v1/models.js'; @@ -20,7 +20,7 @@ async function createActiveStorage() { const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml(PARAMETER_RULES, { storageVersion: storage.STORAGE_VERSION_2 }) ); - const processingStorage = factory.getInstance(syncRules); + const processingStorage = await getTestStorage(factory, syncRules); await using writer = await processingStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.commit('1/1'); diff --git a/modules/module-mongodb-storage/test/src/parameter_compacting_v3.test.ts b/modules/module-mongodb-storage/test/src/parameter_compacting_v3.test.ts index 37e152db2..715c6f667 100644 --- a/modules/module-mongodb-storage/test/src/parameter_compacting_v3.test.ts +++ b/modules/module-mongodb-storage/test/src/parameter_compacting_v3.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { getTestStorage, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoParameterCompactorV3 } from '../../src/storage/implementation/v3/MongoParameterCompactorV3.js'; @@ -31,7 +31,7 @@ async function createActiveStorage(rules = PARAMETER_RULES) { const syncRules = await factory.updateSyncRules( updateSyncRulesFromYaml(rules, { storageVersion: storage.STORAGE_VERSION_3 }) ); - const processingStorage = factory.getInstance(syncRules); + const processingStorage = await getTestStorage(factory, syncRules); await using writer = await processingStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.commit('1/1'); diff --git a/modules/module-mongodb-storage/test/src/parameter_compaction_fence.test.ts b/modules/module-mongodb-storage/test/src/parameter_compaction_fence.test.ts index 4d313ef1d..c2ec8a89b 100644 --- a/modules/module-mongodb-storage/test/src/parameter_compaction_fence.test.ts +++ b/modules/module-mongodb-storage/test/src/parameter_compaction_fence.test.ts @@ -1,6 +1,6 @@ import { mongo } from '@powersync/lib-service-mongodb'; import { InternalOpId, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { getTestStorage, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import type { VersionedPowerSyncMongo } from '../../src/storage/implementation/db.js'; import type { SyncRuleDocumentBase } from '../../src/storage/implementation/models.js'; @@ -83,7 +83,7 @@ describe('parameter compaction invalidation fence', () => { */ async function replicateParameterHistory(factory: storage.BucketStorageFactory) { const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(PARAMETER_RULES, { storageVersion })); - const processingStorage = factory.getInstance(syncRules); + const processingStorage = await getTestStorage(factory, syncRules); const writer = await processingStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -196,7 +196,7 @@ describe('parameter compaction invalidation fence', () => { // which is the only record of t2's lookup. // // A separate storage instance is used to get a cold checkpoint-changes cache. - const coldStorage = (factory as MongoBucketStorage).getInstance(replicationStream); + const coldStorage = await getTestStorage(factory as MongoBucketStorage, replicationStream); const atSnapshot = await coldStorage.getCheckpointChanges({ lastCheckpoint: checkpoint1, nextCheckpoint: checkpoint3 diff --git a/modules/module-mongodb-storage/test/src/storage.test.ts b/modules/module-mongodb-storage/test/src/storage.test.ts index 3159a8de0..14c751faa 100644 --- a/modules/module-mongodb-storage/test/src/storage.test.ts +++ b/modules/module-mongodb-storage/test/src/storage.test.ts @@ -1,7 +1,7 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; import { mongo } from '@powersync/lib-service-mongodb'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { compactActive, register, test_utils } from '@powersync/service-core-tests'; +import { compactActive, getTestStorage, register, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -35,7 +35,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); // user1 has no existing row, so this covers updateMany with upsert. // The initial request stores checkpoint id 42 at source head 5/0. @@ -144,7 +144,8 @@ bucket_definitions: user_id: 'custom1', checkpoint: 52n }); - await writer.flush(); + // A custom-checkpoint-only commit uses the combined final flush path too. + await writer.commit('8/1'); const customGenerated = await factory.db.custom_write_checkpoints.findOne({ user_id: 'custom1' }); expect(customGenerated).not.toBeNull(); expect(customGenerated?.checkpoint_requested_at).toBeUndefined(); @@ -179,7 +180,7 @@ event_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const db = bucketStorage.db as VersionedPowerSyncMongoV3; const activeSyncConfig = bucketStorage.getParsedSyncRules({ defaultSchema: 'public' }); const eventA = activeSyncConfig.eventDescriptors.find((event) => event.name == 'checkpoint_a')!; @@ -228,7 +229,7 @@ event_definitions: writer.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 5n, event_id: eventAId }); writer.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 8n, event_id: eventBId }); - await writer.flush(); + // Collections must be prepared before the combined flush/checkpoint transaction. await writer.keepalive('5/0'); const eventACollection = db.customCheckpointRequests({ @@ -296,7 +297,7 @@ event_definitions: // A fresh storage instance has an empty initialization cache, just like // one created after a service restart. Recreating the existing indexes // must be idempotent before another checkpoint is written. - const freshBucketStorage = factory.getInstance(syncRules); + const freshBucketStorage = await getTestStorage(factory, syncRules); await using freshWriter = await freshBucketStorage.createWriter(test_utils.BATCH_OPTIONS); freshWriter.addCustomWriteCheckpoint({ user_id: 'after-restart', @@ -336,7 +337,7 @@ event_definitions: updateSyncRulesFromYaml(syncConfigYaml("kind = 'write'", false), { storageVersion }) ); const firstEventId = first.syncConfigContent[0].mapping.eventDefinitionIdByName('write_checkpoints'); - const firstStorage = factory.getInstance(first); + const firstStorage = await getTestStorage(factory, first); await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); firstWriter.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 5n, event_id: firstEventId }); await firstWriter.markAllSnapshotDone('1/1'); @@ -348,7 +349,9 @@ event_definitions: const unchangedEventId = unchanged.syncConfigContent[1].mapping.eventDefinitionIdByName('write_checkpoints'); expect(unchangedEventId).toBe(firstEventId); - await using unchangedWriter = await factory.getInstance(unchanged).createWriter(test_utils.BATCH_OPTIONS); + await using unchangedWriter = await ( + await getTestStorage(factory, unchanged) + ).createWriter(test_utils.BATCH_OPTIONS); await unchangedWriter.markAllSnapshotDone('2/1'); await unchangedWriter.commit('2/1'); @@ -369,7 +372,9 @@ event_definitions: const changedEventId = changed.syncConfigContent[1].mapping.eventDefinitionIdByName('write_checkpoints'); expect(changedEventId).not.toBe(unchangedEventId); - await using changedWriter = await factory.getInstance(changed).createWriter(test_utils.BATCH_OPTIONS); + await using changedWriter = await ( + await getTestStorage(factory, changed) + ).createWriter(test_utils.BATCH_OPTIONS); changedWriter.addCustomWriteCheckpoint({ user_id: 'user1', checkpoint: 9n, event_id: changedEventId }); await changedWriter.flush(); @@ -415,7 +420,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); // Replicate an extremely rare possibility where there are multiple records await factory.db.write_checkpoints.insertMany([ diff --git a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts index 59a1f73f0..5509b83b2 100644 --- a/modules/module-mongodb-storage/test/src/storage_compacting.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_compacting.test.ts @@ -17,7 +17,7 @@ import { SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, compactActive, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, getTestStorage, register, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test, vi } from 'vitest'; import { env } from './env.js'; @@ -106,7 +106,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; const { checkpoint } = await populate(bucketStorage, 1); @@ -164,7 +164,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await populate(bucketStorage, 2); @@ -356,7 +356,7 @@ bucket_definitions: { storageVersion: 3 } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const mapping = syncRules.syncConfigContent[0].mapping; const definitionId = mapping.allBucketDefinitionIds()[0]; @@ -990,7 +990,7 @@ bucket_definitions: { storageVersion: 3 } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const mapping = syncRules.syncConfigContent[0].mapping; const definitionId = mapping.allBucketDefinitionIds()[0]; @@ -1167,7 +1167,7 @@ bucket_definitions: { storageVersion: 3 } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const mapping = syncRules.syncConfigContent[0].mapping; const definitionId = mapping.allBucketDefinitionIds()[0]; @@ -1560,7 +1560,7 @@ bucket_definitions: { storageVersion: 3 } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const mapping = syncRules.syncConfigContent[0].mapping; const definitionId = mapping.allBucketDefinitionIds()[0]; @@ -1749,7 +1749,7 @@ bucket_definitions: { storageVersion: 3 } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const definitionId = bucketStorage.storageIds.bucketDefinitionIds[0]; const collection = db.bucketData(bucketStorage.replicationStreamId, definitionId); diff --git a/modules/module-mongodb-storage/test/src/storage_object_storage_inline_threshold.test.ts b/modules/module-mongodb-storage/test/src/storage_object_storage_inline_threshold.test.ts index eb614949b..37084f9d8 100644 --- a/modules/module-mongodb-storage/test/src/storage_object_storage_inline_threshold.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_object_storage_inline_threshold.test.ts @@ -1,6 +1,6 @@ import { mongoTestStorageFactoryGenerator } from '@module/utils/test-utils.js'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, getTestStorage, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { DEFAULT_INLINE_THRESHOLD_BYTES } from '../../src/storage/implementation/common/PersistedBatch.js'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; @@ -31,7 +31,7 @@ describe('Object storage inline threshold', () => { const { memoryStorage, factoryGen } = s3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; expect(bucketStorage.inlineThresholdBytes).toBe(DEFAULT_INLINE_THRESHOLD_BYTES); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -77,7 +77,7 @@ describe('Object storage inline threshold', () => { const { memoryStorage, factoryGen } = s3Factory(256); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; expect(bucketStorage.inlineThresholdBytes).toBe(256); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts index aaaa8ee8d..7682a5fe1 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_checksums.test.ts @@ -4,7 +4,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, getTestStorage, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { VersionedPowerSyncMongoV3 } from '../../src/storage/implementation/v3/VersionedPowerSyncMongoV3.js'; @@ -28,7 +28,7 @@ describe('V3 checksums with S3 object storage', () => { const { factoryGen } = s3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); @@ -75,7 +75,7 @@ describe('V3 checksums with S3 object storage', () => { const { factoryGen } = s3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); @@ -114,7 +114,7 @@ describe('V3 checksums with S3 object storage', () => { const { factoryGen } = s3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const request = bucketRequest(syncRules.syncConfigContent[0], 'global[]', 0n); diff --git a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts index 9fcf9c52e..f07e8629d 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_compaction_lifecycle.test.ts @@ -1,6 +1,6 @@ import * as lib_mongo from '@powersync/lib-service-mongodb'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, compactActive, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, compactActive, getTestStorage, test_utils } from '@powersync/service-core-tests'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; import { MongoSyncBucketStorageV3 } from '../../src/storage/implementation/v3/MongoSyncBucketStorageV3.js'; @@ -57,7 +57,7 @@ describe('S3 compaction storage lifecycle', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1); @@ -106,7 +106,7 @@ describe('S3 compaction storage lifecycle', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory({ inlineThresholdBytes: 10_000 }); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 10); @@ -152,7 +152,7 @@ describe('S3 compaction storage lifecycle', () => { const { factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 11); @@ -187,7 +187,7 @@ describe('S3 compaction storage lifecycle', () => { const { factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 12); @@ -248,7 +248,7 @@ describe('S3 compaction storage lifecycle', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1); @@ -323,7 +323,7 @@ describe('S3 compaction storage lifecycle', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1); @@ -396,7 +396,7 @@ describe('S3 compaction storage lifecycle', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; // Write several operations, including a repeated object id, to exercise // the compaction round-trip. diff --git a/modules/module-mongodb-storage/test/src/storage_s3_reading.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_reading.test.ts index 79d59844b..3c09355a2 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_reading.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_reading.test.ts @@ -1,6 +1,6 @@ import { DeleteObjectsCommand, ListObjectsV2Command } from '@aws-sdk/client-s3'; import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, getTestStorage, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test, vi } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; @@ -662,7 +662,7 @@ describe('S3 object storage reads', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1); @@ -710,7 +710,7 @@ describe('S3 object storage reads', () => { const { factory: factoryGen } = s3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1); @@ -763,7 +763,7 @@ describe('S3 object storage reads', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory({ inlineThresholdBytes: 1_000 }); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1); diff --git a/modules/module-mongodb-storage/test/src/storage_s3_writing.test.ts b/modules/module-mongodb-storage/test/src/storage_s3_writing.test.ts index c55646c8b..f5aa090a3 100644 --- a/modules/module-mongodb-storage/test/src/storage_s3_writing.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_s3_writing.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { test_utils } from '@powersync/service-core-tests'; +import { getTestStorage, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoSyncBucketStorage } from '../../src/storage/implementation/createMongoSyncBucketStorage.js'; @@ -28,7 +28,7 @@ describe('S3 object storage writes', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 1); @@ -95,7 +95,7 @@ describe('S3 object storage writes', () => { const { memoryStorage, factory: factoryGen } = memoryS3Factory(); await using factory = await factoryGen.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES_YAML, { storageVersion: 3 })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'items', ['id'], factoryGen, 4); diff --git a/modules/module-mongodb-storage/test/src/storage_sync.test.ts b/modules/module-mongodb-storage/test/src/storage_sync.test.ts index e73c3d7c3..a7f98a9db 100644 --- a/modules/module-mongodb-storage/test/src/storage_sync.test.ts +++ b/modules/module-mongodb-storage/test/src/storage_sync.test.ts @@ -6,7 +6,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, register, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, getTestStorage, register, test_utils } from '@powersync/service-core-tests'; import { RequestParameters, ScopedParameterLookup, SqlSyncRules } from '@powersync/service-sync-rules'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; @@ -149,7 +149,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor test('updates source metadata on an existing resolved table', async () => { await using factory = await storageConfig.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const source = sourceDescriptor('test'); @@ -191,7 +191,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -338,7 +338,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -363,7 +363,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const before = await writer.resolveTables({ @@ -401,7 +401,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const before = await writer.resolveTables({ @@ -437,7 +437,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const before = await writer.resolveTables({ @@ -481,7 +481,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await writer.resolveTables({ @@ -550,7 +550,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor // Here we're persisting one sync config, then resolving tables with others. // We're also using the default hydration state for them all. const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(fullRulesYaml, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const fullRules = parsedSyncConfigSetFor(fullRulesYaml, storageVersion); const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyRulesYaml, storageVersion); @@ -666,7 +666,7 @@ function registerSyncStorageTests(storageConfig: storage.TestStorageConfig, stor await using factory = await storageConfig.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(dataOnlyEventYaml, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const dataOnlyRules = parsedSyncConfigSetFor(dataOnlyEventYaml, storageVersion); const fullRules = parsedSyncConfigSetFor(fullEventYaml, storageVersion); @@ -739,7 +739,7 @@ event_definitions: }); try { const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstYaml, { storageVersion })); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' }); const firstResolved = await firstWriter.resolveTables({ @@ -755,7 +755,7 @@ event_definitions: const second = await factory.updateSyncRules(updateSyncRulesFromYaml(secondYaml, { storageVersion })); expect(second.replicationStreamId).toBe(first.replicationStreamId); - const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage; + const secondStorage = (await getTestStorage(factory, second)) as MongoSyncBucketStorage; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await secondWriter.resolveTables({ connection_id: 1, @@ -806,7 +806,7 @@ event_definitions: }); try { const first = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(true), { storageVersion })); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); const source = sourceDescriptor('checkpoints', { objectId: 'checkpoints-relation' }); const firstResolved = await firstWriter.resolveTables({ @@ -821,7 +821,7 @@ event_definitions: const second = await factory.updateSyncRules(updateSyncRulesFromYaml(yaml(false), { storageVersion })); expect(second.replicationStreamId).toBe(first.replicationStreamId); - const secondStorage = factory.getInstance(second) as MongoSyncBucketStorage; + const secondStorage = (await getTestStorage(factory, second)) as MongoSyncBucketStorage; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await secondWriter.resolveTables({ connection_id: 1, @@ -875,7 +875,7 @@ event_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; const sync_rules = syncRulesContent.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -988,7 +988,7 @@ streams: expect(parsed.syncConfigs).toHaveLength(1); expect(parsed.hydratedSyncConfig.bucketDataSources).toHaveLength(1); - const bucketStorage = factory.getInstance(replicatingStreams[0]); + const bucketStorage = await getTestStorage(factory, replicatingStreams[0]); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await writer.resolveTables({ connection_id: 1, @@ -1015,7 +1015,7 @@ streams: { storageVersion } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1035,7 +1035,7 @@ streams: ); expect(second.replicationStreamId).toEqual(first.replicationStreamId); - const bucketStorage = factory.getInstance(second); + const bucketStorage = await getTestStorage(factory, second); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const source = sourceDescriptor('todos', { objectId: 'todos-relation' }); @@ -1050,7 +1050,10 @@ streams: await writer.markAllSnapshotDone('2/1'); await writer.commit('2/1'); - const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage; + const activeStorage = (await getTestStorage( + factory, + (await factory.getActiveSyncConfig())!.replicationStream + )) as MongoSyncBucketStorage; await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS); const activeStatus = await activeWriter.getSourceTableStatus(resolved.tables[0]); expect(activeStatus?.bucketDataSources).toHaveLength(1); @@ -1074,7 +1077,7 @@ streams: { storageVersion } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); const source = sourceDescriptor('todos', { objectId: 'todos-relation' }); const firstResolved = await firstWriter.resolveTables({ @@ -1103,7 +1106,7 @@ streams: const deploying = await factory.getDeployingSyncConfig(); expect(deploying?.replicationStream.replicationStreamId).toBe(first.replicationStreamId); - const bucketStorage = deploying!.storage; + const bucketStorage = await getTestStorage(factory, deploying!.replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await writer.resolveTables({ connection_id: 1, @@ -1149,7 +1152,7 @@ streams: `; const first = await factory.updateSyncRules(updateSyncRulesFromYaml(ownerRules, { storageVersion })); - const firstStorage = factory.getInstance(first); + const firstStorage = await getTestStorage(factory, first); await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1201,7 +1204,7 @@ streams: { storageVersion } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1223,7 +1226,7 @@ streams: ); expect(second.replicationStreamId).toBe(first.replicationStreamId); - const bucketStorage = factory.getInstance(second); + const bucketStorage = await getTestStorage(factory, second); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const resolved = await writer.resolveTables({ connection_id: 1, @@ -1274,7 +1277,7 @@ streams: { storageVersion } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1317,7 +1320,7 @@ streams: expect(replicatingStreams[0].replicationJobId).toContain(config.syncConfigId); } - const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage; + const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorage; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); await secondWriter.markAllSnapshotDone('2/1'); await secondWriter.commit('2/1'); @@ -1360,7 +1363,7 @@ streams: { storageVersion } ) ); - const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage; + const activeStorage = (await getTestStorage(factory, active)) as MongoSyncBucketStorage; await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS); await activeWriter.markAllSnapshotDone('1/1'); await activeWriter.commit('1/1'); @@ -1396,7 +1399,7 @@ streams: { storageVersion } ) ); - const activeStorage = factory.getInstance(active) as MongoSyncBucketStorage; + const activeStorage = (await getTestStorage(factory, active)) as MongoSyncBucketStorage; await using activeWriter = await activeStorage.createWriter(test_utils.BATCH_OPTIONS); await activeWriter.markAllSnapshotDone('1/1'); await activeWriter.commit('1/1'); @@ -1455,7 +1458,7 @@ streams: { storageVersion } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable( firstWriter, @@ -1505,7 +1508,7 @@ streams: const replicatingStreams = await factory.getReplicatingReplicationStreams(); expect(replicatingStreams).toHaveLength(1); - const secondStorage = factory.getInstance(replicatingStreams[0]) as MongoSyncBucketStorage; + const secondStorage = (await getTestStorage(factory, replicatingStreams[0])) as MongoSyncBucketStorage; await using secondWriter = await secondStorage.createWriter(test_utils.BATCH_OPTIONS); // No new data replicated - just complete the snapshot and commit. await secondWriter.markAllSnapshotDone('2/1'); @@ -1524,7 +1527,10 @@ streams: expect(head).toBeGreaterThanOrEqual(firstCheckpoint); // After activation, the active config's checkpoint does not regress. - const activeStorage = (await factory.getActiveSyncConfig())?.storage as MongoSyncBucketStorage; + const activeStorage = (await getTestStorage( + factory, + (await factory.getActiveSyncConfig())!.replicationStream + )) as MongoSyncBucketStorage; const activeCheckpoint = (await activeStorage.getCheckpoint()).checkpoint; expect(activeCheckpoint).toBeGreaterThanOrEqual(firstCheckpoint); } @@ -1546,7 +1552,7 @@ streams: { storageVersion } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1593,7 +1599,7 @@ streams: { storageVersion } ) ); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1608,7 +1614,7 @@ streams: await using factory = await storageConfig.factory(); const first = await factory.updateSyncRules(updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion })); - const firstStorage = factory.getInstance(first) as MongoSyncBucketStorage; + const firstStorage = (await getTestStorage(factory, first)) as MongoSyncBucketStorage; await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1681,7 +1687,7 @@ streams: const first = await factory.updateSyncRules(updateSyncRulesFromYaml(firstRules, { storageVersion, lock: true })); expect(first.current_lock?.sync_rules_id).toBe(first.replicationStreamId); try { - const firstStorage = factory.getInstance(first); + const firstStorage = factory.getInstance(first, { replicationLock: first.current_lock! }); await using firstWriter = await firstStorage.createWriter(test_utils.BATCH_OPTIONS); await firstWriter.markAllSnapshotDone('1/1'); await firstWriter.commit('1/1'); @@ -1709,7 +1715,7 @@ streams: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -1748,7 +1754,7 @@ streams: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], INITIALIZED_MONGO_STORAGE_FACTORY); @@ -1782,7 +1788,7 @@ streams: storageVersion }) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const mongoFactory = factory as MongoBucketStorage; const sourceTableId = new bson.ObjectId(); const documents = Array.from({ length: 10_002 }, (_, index) => ({ @@ -1825,7 +1831,7 @@ streams: storageVersion }) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const mongoFactory = factory as MongoBucketStorage; const sourceTableId = new bson.ObjectId(); const documents = Array.from({ length: 10_002 }, (_, index) => ({ @@ -1876,7 +1882,7 @@ streams: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const metricsBefore = await factory.getStorageMetrics(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -1939,7 +1945,7 @@ streams: { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const previousCheckpoint = await bucketStorage.getCheckpoint(); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -1984,7 +1990,7 @@ streams: ); const mongoFactory = factory as MongoBucketStorage; - const bucketStorage = mongoFactory.getInstance(syncRules); + const bucketStorage = await getTestStorage(mongoFactory, syncRules); const db = bucketStorage.db as VersionedPowerSyncMongoV3; await db.initializeStreamStorage(syncRules.replicationStreamId); @@ -2079,7 +2085,7 @@ describe('sync - mongodb', () => { const legacySyncRules = await factory.updateSyncRules( updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.LEGACY_STORAGE_VERSION }) ); - const legacyStorage = factory.getInstance(legacySyncRules); + const legacyStorage = await getTestStorage(factory, legacySyncRules); await using legacyWriter = await legacyStorage.createWriter(test_utils.BATCH_OPTIONS); await legacyWriter.markAllSnapshotDone('1/1'); await legacyWriter.commit('1/1'); @@ -2091,7 +2097,7 @@ describe('sync - mongodb', () => { const v3SyncRules = await factory.updateSyncRules( updateSyncRulesFromYaml(MINIMAL_SYNC_RULES, { storageVersion: storage.STORAGE_VERSION_3 }) ); - const v3Storage = factory.getInstance(v3SyncRules); + const v3Storage = await getTestStorage(factory, v3SyncRules); await using v3Writer = await v3Storage.createWriter(test_utils.BATCH_OPTIONS); await v3Writer.markAllSnapshotDone('2/1'); await v3Writer.commit('2/1'); @@ -2119,7 +2125,7 @@ describe('sync - mongodb', () => { { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorage; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorage; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const mapping = syncRules.syncConfigContent[0].mapping; @@ -2251,7 +2257,7 @@ describe('sync - mongodb', () => { { storageVersion } ) ); - const bucketStorage = factory.getInstance(syncRules) as MongoSyncBucketStorageV3; + const bucketStorage = (await getTestStorage(factory, syncRules)) as MongoSyncBucketStorageV3; const db = bucketStorage.db as VersionedPowerSyncMongoV3; const start = 5n; diff --git a/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts b/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts index 2460f93e4..1b1161a8e 100644 --- a/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts +++ b/modules/module-mongodb-storage/test/src/storeCurrentData.test.ts @@ -1,5 +1,5 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, getTestStorage, test_utils } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { describe, expect, test } from 'vitest'; import { MongoBucketStorage } from '../../src/storage/MongoBucketStorage.js'; @@ -80,7 +80,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { test('resolveTables derives storeCurrentData fresh each call, with no persisted memory', async () => { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -114,7 +114,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { test('storeCurrentData=false omits the row payload from current_data, data still syncs', async () => { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -146,7 +146,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { test('storeCurrentData=true retains the row payload in current_data', async () => { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -179,7 +179,7 @@ function registerStoreCurrentDataTests(storageVersion: number) { test('storeCurrentData=false processes UPDATE without a stored copy', async () => { await using factory = await INITIALIZED_MONGO_STORAGE_FACTORY.factory(); const syncRules = await factory.updateSyncRules(updateSyncRulesFromYaml(SYNC_RULES, { storageVersion })); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/modules/module-mongodb/test/src/change_stream_utils.ts b/modules/module-mongodb/test/src/change_stream_utils.ts index ce15586e0..4ce92c0c4 100644 --- a/modules/module-mongodb/test/src/change_stream_utils.ts +++ b/modules/module-mongodb/test/src/change_stream_utils.ts @@ -18,7 +18,7 @@ import { updateSyncRulesFromYaml, utils } from '@powersync/service-core'; -import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, getTestStorage, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import { SentinelLSN } from '@module/common/SentinelLSN.js'; import { ChangeStream, ChangeStreamOptions } from '@module/replication/ChangeStream.js'; @@ -119,7 +119,7 @@ export class ChangeStreamTestContext { updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); this.syncRulesContent = replicationStream.syncConfigContent[0]; - this.storage = this.factory.getInstance(replicationStream); + this.storage = await getTestStorage(this.factory, replicationStream); return this.storage!; } @@ -130,7 +130,7 @@ export class ChangeStreamTestContext { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } @@ -141,7 +141,7 @@ export class ChangeStreamTestContext { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } diff --git a/modules/module-mssql/test/src/CDCStreamTestContext.ts b/modules/module-mssql/test/src/CDCStreamTestContext.ts index 0e38d9248..7f3ebe0a6 100644 --- a/modules/module-mssql/test/src/CDCStreamTestContext.ts +++ b/modules/module-mssql/test/src/CDCStreamTestContext.ts @@ -11,7 +11,7 @@ import { SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, getTestStorage, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import timers from 'timers/promises'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -86,7 +86,7 @@ export class CDCStreamTestContext implements AsyncDisposable { updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION }) ); this.syncRulesContent = replicationStream.syncConfigContent[0]; - this.storage = this.factory.getInstance(replicationStream); + this.storage = await getTestStorage(this.factory, replicationStream); return this.storage!; } @@ -97,7 +97,7 @@ export class CDCStreamTestContext implements AsyncDisposable { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } @@ -108,7 +108,7 @@ export class CDCStreamTestContext implements AsyncDisposable { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } diff --git a/modules/module-mysql/test/src/BinlogStreamUtils.ts b/modules/module-mysql/test/src/BinlogStreamUtils.ts index c9f1a0723..e4b14e2dc 100644 --- a/modules/module-mysql/test/src/BinlogStreamUtils.ts +++ b/modules/module-mysql/test/src/BinlogStreamUtils.ts @@ -15,7 +15,7 @@ import { SyncRulesBucketStorage, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; +import { bucketRequest, getTestStorage, METRICS_HELPER, test_utils } from '@powersync/service-core-tests'; import mysqlPromise from 'mysql2/promise'; import timers from 'timers/promises'; import { clearTestDb, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -75,7 +75,7 @@ export class BinlogStreamTestContext { updateSyncRulesFromYaml(content, { validate: true, storageVersion: LEGACY_STORAGE_VERSION }) ); this.syncRulesContent = replicationStream.syncConfigContent[0]; - this.storage = this.factory.getInstance(replicationStream); + this.storage = await getTestStorage(this.factory, replicationStream); return this.storage!; } @@ -86,7 +86,7 @@ export class BinlogStreamTestContext { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } @@ -97,7 +97,7 @@ export class BinlogStreamTestContext { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); this.replicationDone = true; return this.storage!; } diff --git a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts index 0a56f1d61..269e3f60b 100644 --- a/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts +++ b/modules/module-postgres-storage/src/storage/PostgresBucketStorageFactory.ts @@ -41,7 +41,11 @@ export class PostgresBucketStorageFactory extends storage.BucketStorageFactory { } async [Symbol.asyncDispose]() { - await this.db[Symbol.asyncDispose](); + try { + await super[Symbol.asyncDispose](); + } finally { + await this.db[Symbol.asyncDispose](); + } } async prepareStatements(connection: pg_wire.PgConnection) { diff --git a/modules/module-postgres/test/src/resuming_snapshots.test.ts b/modules/module-postgres/test/src/resuming_snapshots.test.ts index db981f843..bc989967e 100644 --- a/modules/module-postgres/test/src/resuming_snapshots.test.ts +++ b/modules/module-postgres/test/src/resuming_snapshots.test.ts @@ -64,25 +64,28 @@ async function testResumingReplication( let done = false; const startRowCount = (await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0; - try { - (async () => { - while (!done) { - const count = - ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; - - if (count >= stopAfter) { - break; - } - await timers.setTimeout(1); + const stopReplication = (async () => { + while (!done) { + const count = + ((await METRICS_HELPER.getMetricValueForTests(ReplicationMetric.ROWS_REPLICATED)) ?? 0) - startRowCount; + + if (count >= stopAfter) { + break; } - // This interrupts initial replication - await context.dispose(); - })(); + await timers.setTimeout(1); + } + // This interrupts initial replication + await context.dispose(); + })(); + try { // This confirms that initial replication was interrupted await expect(p).rejects.toThrowError(); done = true; } finally { done = true; + // Replication rejects before dispose finishes closing connections and + // releasing the lease. Wait before the replacement tries to acquire it. + await stopReplication; } // Bypass the usual "clear db on factory open" step. diff --git a/modules/module-postgres/test/src/slow_tests.test.ts b/modules/module-postgres/test/src/slow_tests.test.ts index 832508549..d551e1e62 100644 --- a/modules/module-postgres/test/src/slow_tests.test.ts +++ b/modules/module-postgres/test/src/slow_tests.test.ts @@ -1,3 +1,4 @@ +import { getTestStorage } from '@powersync/service-core-tests'; import * as bson from 'bson'; import { afterEach, beforeAll, describe, expect, test } from 'vitest'; import { WalStream, WalStreamOptions } from '../../src/replication/WalStream.js'; @@ -96,7 +97,7 @@ bucket_definitions: - SELECT * FROM "test_data" `; const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion })); - const storage = f.getInstance(syncRules); + const storage = await getTestStorage(f, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; const helpers = new StorageDataHelpers(storage, syncRulesContent); abortController = new AbortController(); @@ -297,7 +298,7 @@ bucket_definitions: `; const syncRules = await f.updateSyncRules(updateSyncRulesFromYaml(syncRuleContent, { storageVersion })); - const storage = f.getInstance(syncRules); + const storage = await getTestStorage(f, syncRules); // 1. Setup some base data that will be replicated in initial replication await pool.query(`CREATE TABLE test_data(id uuid primary key default uuid_generate_v4(), description text)`); diff --git a/modules/module-postgres/test/src/wal_stream_utils.ts b/modules/module-postgres/test/src/wal_stream_utils.ts index 8e42caa10..9519e6b6a 100644 --- a/modules/module-postgres/test/src/wal_stream_utils.ts +++ b/modules/module-postgres/test/src/wal_stream_utils.ts @@ -13,7 +13,13 @@ import { unsettledPromise, updateSyncRulesFromYaml } from '@powersync/service-core'; -import { bucketRequest, METRICS_HELPER, StorageDataHelpers, test_utils } from '@powersync/service-core-tests'; +import { + bucketRequest, + getTestStorage, + METRICS_HELPER, + StorageDataHelpers, + test_utils +} from '@powersync/service-core-tests'; import * as pgwire from '@powersync/service-jpgwire'; import { clearTestDb, getClientCheckpoint, TEST_CONNECTION_OPTIONS } from './util.js'; @@ -93,7 +99,7 @@ export class WalStreamTestContext implements AsyncDisposable { updateSyncRulesFromYaml(content, { validate: true, storageVersion: this.storageVersion }) ); this.syncRulesContent = replicationStream.syncConfigContent[0]; - this.storage = this.factory.getInstance(replicationStream); + this.storage = await getTestStorage(this.factory, replicationStream); return this.storage!; } @@ -104,7 +110,7 @@ export class WalStreamTestContext implements AsyncDisposable { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } @@ -115,7 +121,7 @@ export class WalStreamTestContext implements AsyncDisposable { } this.syncRulesContent = syncConfig.content; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } diff --git a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts index f038e6b9e..7d2a41e1e 100644 --- a/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts +++ b/packages/service-core-tests/src/test-utils/AbstractStreamTestContext.ts @@ -11,6 +11,7 @@ import { } from '@powersync/service-core'; import { StorageDataHelpers } from './StorageDataHelpers.js'; import { bucketRequest, getBatchArray } from './general-utils.js'; +import { getTestStorage } from './leased-storage.js'; export abstract class AbstractStreamTestContext implements AsyncDisposable { protected abortController = new AbortController(); @@ -50,7 +51,7 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable { ); this.replicationStream = stream; this.syncRulesContent = stream.syncConfigContent[0]; - this.storage = this.factory.getInstance(stream); + this.storage = await getTestStorage(this.factory, stream); return this.storage!; } @@ -62,7 +63,7 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable { this.syncRulesContent = syncConfig.content; this.replicationStream = syncConfig.replicationStream; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } @@ -74,7 +75,7 @@ export abstract class AbstractStreamTestContext implements AsyncDisposable { this.syncRulesContent = syncConfig.content; this.replicationStream = syncConfig.replicationStream; - this.storage = syncConfig.storage; + this.storage = await getTestStorage(this.factory, syncConfig.replicationStream); return this.storage!; } diff --git a/packages/service-core-tests/src/test-utils/leased-storage.ts b/packages/service-core-tests/src/test-utils/leased-storage.ts new file mode 100644 index 000000000..62ba0614d --- /dev/null +++ b/packages/service-core-tests/src/test-utils/leased-storage.ts @@ -0,0 +1,58 @@ +import { storage } from '@powersync/service-core'; + +const leases = new WeakMap>>(); + +/** + * Acquire a real replication lease before constructing writable test storage. + * Writers in the same test job share a lease per factory/stream. The factory owns + * these leases and releases them at teardown; takeover tests manage their own leases. + */ +export async function getTestStorage( + factory: T, + stream: storage.PersistedReplicationStream, + options?: Parameters[1] +): Promise> { + let owned = leases.get(factory); + if (owned == null) { + owned = new Map(); + leases.set(factory, owned); + const factoryLeases = owned; + // `await using` captures the disposer before this helper runs. Register a + // lifecycle callback instead of replacing that already-captured method. + const unregister = factory.registerListener({ + beforeDispose: async () => { + unregister(); + try { + for (const lock of factoryLeases.values()) { + await (await lock).release(); + } + } finally { + factoryLeases.clear(); + } + } + }); + } + let pending = owned.get(stream.replicationStreamId); + if (pending == null) { + pending = stream.lock(); + owned.set(stream.replicationStreamId, pending); + } + let lock: storage.ReplicationLock; + try { + lock = await pending; + } catch (error) { + owned.delete(stream.replicationStreamId); + throw error; + } + return factory.getInstance(stream, { ...options, replicationLock: lock }) as ReturnType; +} + +/** End a test job before starting a replacement through another factory. */ +export async function releaseTestStorageLease(factory: storage.BucketStorageFactory, streamId: number) { + const owned = leases.get(factory); + const pending = owned?.get(streamId); + if (pending != null) { + await (await pending).release(); + owned!.delete(streamId); + } +} diff --git a/packages/service-core-tests/src/test-utils/test-utils-index.ts b/packages/service-core-tests/src/test-utils/test-utils-index.ts index 1d48f239c..1e95d63d0 100644 --- a/packages/service-core-tests/src/test-utils/test-utils-index.ts +++ b/packages/service-core-tests/src/test-utils/test-utils-index.ts @@ -1,6 +1,7 @@ export * from './AbstractStreamTestContext.js'; export * from './bucket-validation.js'; export * from './general-utils.js'; +export * from './leased-storage.js'; export * from './MetricsHelper.js'; export * from './storage-combinations.js'; export * from './StorageDataHelpers.js'; diff --git a/packages/service-core-tests/src/tests/register-compacting-tests.ts b/packages/service-core-tests/src/tests/register-compacting-tests.ts index 916952855..2d68cf7f9 100644 --- a/packages/service-core-tests/src/tests/register-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-compacting-tests.ts @@ -1,5 +1,6 @@ import { addChecksums, storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { expect, test } from 'vitest'; +import { getTestStorage } from '../test-utils/leased-storage.js'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; import { bucketRequestMap, bucketRequests, compactActive } from './util.js'; @@ -16,7 +17,7 @@ bucket_definitions: data: [select * from test] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -128,7 +129,7 @@ bucket_definitions: data: [select * from test] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -249,7 +250,7 @@ bucket_definitions: data: [select * from test] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -340,7 +341,7 @@ bucket_definitions: data: - select * from test where b = bucket.b`) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -471,7 +472,7 @@ bucket_definitions: data: [select * from test] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -550,7 +551,7 @@ bucket_definitions: data: [select * from test] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -625,7 +626,7 @@ bucket_definitions: data: [select * from test] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); const syncRulesContent = syncRules.syncConfigContent[0]; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts index 0cd55d336..9859ee2b6 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-checkpoint-tests.ts @@ -1,5 +1,6 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { expect, test } from 'vitest'; +import { getTestStorage } from '../test-utils/leased-storage.js'; import * as test_utils from '../test-utils/test-utils-index.js'; import { compactActive } from './util.js'; @@ -32,7 +33,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(r.persisted_sync_rules!); + const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!); const abortController = new AbortController(); context.onTestFinished(() => abortController.abort()); @@ -81,7 +82,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(r.persisted_sync_rules!); + const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); @@ -150,7 +151,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(r.persisted_sync_rules!); + const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!); const first = await createManagedWriteCheckpointResult(bucketStorage, { user_id: 'user1', @@ -216,7 +217,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(r.persisted_sync_rules!); + const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode({ mode: storage.WriteCheckpointMode.CUSTOM }); @@ -267,7 +268,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(r.persisted_sync_rules!); + const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode({ mode: storage.WriteCheckpointMode.CUSTOM }); @@ -321,7 +322,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(r.persisted_sync_rules!); + const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode({ mode: storage.WriteCheckpointMode.CUSTOM }); @@ -407,7 +408,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(r.persisted_sync_rules!); + const bucketStorage = await getTestStorage(factory, r.persisted_sync_rules!); bucketStorage.setWriteCheckpointMode({ mode: storage.WriteCheckpointMode.CUSTOM }); diff --git a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts index 28de0c7d7..669fd3ec9 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-data-tests.ts @@ -8,6 +8,7 @@ import { updateSyncRulesFromYaml } from '@powersync/service-core'; import { describe, expect, test } from 'vitest'; +import { getTestStorage } from '../test-utils/leased-storage.js'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; @@ -36,6 +37,21 @@ export function registerDataStorageDataTests(config: storage.TestStorageConfig) const generateStorageFactory = config.factory; const storageVersion = config.storageVersion ?? storage.CURRENT_STORAGE_VERSION; + test('releases test replication leases when leaving an await using scope', async () => { + { + await using factory = await generateStorageFactory(); + const stream = await factory.updateSyncRules( + updateSyncRulesFromYaml('bucket_definitions: {}', { storageVersion }) + ); + await getTestStorage(factory, stream); + } + + await using replacement = await generateStorageFactory({ doNotClear: true }); + const deploying = await replacement.getDeployingSyncConfig(); + expect(deploying).not.toBeNull(); + await getTestStorage(replacement, deploying!.replicationStream); + }); + test('removing row', async () => { await using factory = await generateStorageFactory(); const { stream: replicationStream, content: syncRules } = await test_utils.deploySyncRules( @@ -50,7 +66,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -116,7 +132,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -181,7 +197,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -251,7 +267,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -314,7 +330,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); { await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -393,7 +409,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -465,7 +481,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -540,7 +556,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -671,7 +687,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -835,7 +851,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config); @@ -949,7 +965,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id', 'description'], config); @@ -1053,7 +1069,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -1167,7 +1183,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -1267,7 +1283,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); @@ -1466,7 +1482,7 @@ bucket_definitions: }); const r = await f.configureSyncRules(updateSyncRulesFromYaml('bucket_definitions: {}')); - const storage = f.getInstance(r.persisted_sync_rules!); + const storage = await getTestStorage(f, r.persisted_sync_rules!); await using writer = await storage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/0'); await writer.keepalive('1/0'); @@ -1495,7 +1511,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, 1); @@ -1547,7 +1563,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -1589,7 +1605,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('1/1'); await writer.commit('1/1'); @@ -1628,7 +1644,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer1 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await using writer2 = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer2, 'test', ['id'], config); @@ -1679,7 +1695,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const result1 = await writer.commit('1/1', { createEmptyCheckpoints: false }); @@ -1732,7 +1748,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using snapshotWriter = await bucketStorage.createWriter({ ...test_utils.BATCH_OPTIONS, skipExistingRows: true @@ -1803,7 +1819,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); await writer.markAllSnapshotDone('1/1'); diff --git a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts index ea92ec24a..420e4aca7 100644 --- a/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts +++ b/packages/service-core-tests/src/tests/register-data-storage-parameter-tests.ts @@ -7,6 +7,7 @@ import { UnscopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; +import { getTestStorage } from '../test-utils/leased-storage.js'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest } from '../test-utils/test-utils-index.js'; @@ -41,7 +42,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -108,7 +109,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -180,7 +181,7 @@ bucket_definitions: { storageVersion } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -268,7 +269,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -337,7 +338,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -407,7 +408,7 @@ bucket_definitions: ) ); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', ['id'], config); @@ -466,7 +467,7 @@ bucket_definitions: ) ); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); @@ -561,7 +562,7 @@ bucket_definitions: ) ); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const workspaceTable = await test_utils.resolveTestTable(writer, 'workspace', undefined, config); @@ -662,7 +663,7 @@ bucket_definitions: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -716,7 +717,7 @@ bucket_definitions: } ) ); - const syncBucketStorage = bucketStorageFactory.getInstance(syncRules); + const syncBucketStorage = await getTestStorage(bucketStorageFactory, syncRules); const parsedSchema1 = syncBucketStorage.getParsedSyncRules({ defaultSchema: 'public' @@ -754,7 +755,7 @@ streams: WHERE data.foo = param.bar AND param.baz = auth.user_id() `) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -826,7 +827,7 @@ streams: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -876,7 +877,7 @@ streams: WHERE a.x = x.value AND y.value = auth.user_id() `) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); @@ -1005,7 +1006,7 @@ streams: query: SELECT * FROM b WHERE p IN (SELECT id FROM param_b WHERE u = auth.user_id()) `) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const parsedSyncRules = syncRules.parsed(test_utils.PARSE_OPTIONS); const hydrationState = parsedSyncRules.hydrationState; const [parsedSyncConfig] = parsedSyncRules.syncConfigs; @@ -1106,7 +1107,7 @@ streams: } ) ); - const bucketStorage = factory.getInstance(replicationStream); + const bucketStorage = await getTestStorage(factory, replicationStream); const sync_rules = syncRules.parsed(test_utils.PARSE_OPTIONS).hydratedSyncConfig; await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); diff --git a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts index ff67ada62..5da154166 100644 --- a/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts +++ b/packages/service-core-tests/src/tests/register-parameter-compacting-tests.ts @@ -1,6 +1,7 @@ import { storage, updateSyncRulesFromYaml } from '@powersync/service-core'; import { ScopedParameterLookup } from '@powersync/service-sync-rules'; import { expect, test } from 'vitest'; +import { getTestStorage } from '../test-utils/leased-storage.js'; import * as test_utils from '../test-utils/test-utils-index.js'; import { compactActive } from './util.js'; @@ -17,7 +18,7 @@ bucket_definitions: data: [] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -99,7 +100,7 @@ bucket_definitions: data: [] `) ); - const bucketStorage = factory.getInstance(syncRules); + const bucketStorage = await getTestStorage(factory, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); diff --git a/packages/service-core-tests/src/tests/register-sync-tests.ts b/packages/service-core-tests/src/tests/register-sync-tests.ts index f3387f1b3..0776c5591 100644 --- a/packages/service-core-tests/src/tests/register-sync-tests.ts +++ b/packages/service-core-tests/src/tests/register-sync-tests.ts @@ -13,6 +13,7 @@ import path from 'path'; import * as timers from 'timers/promises'; import { fileURLToPath } from 'url'; import { expect, test, vi } from 'vitest'; +import { getTestStorage } from '../test-utils/leased-storage.js'; import * as test_utils from '../test-utils/test-utils-index.js'; import { bucketRequest, METRICS_HELPER } from '../test-utils/test-utils-index.js'; import { compactActive } from './util.js'; @@ -70,7 +71,7 @@ export function registerSyncTests( content: BASIC_SYNC_RULES }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const sourceTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -133,7 +134,7 @@ bucket_definitions: ` }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -192,7 +193,7 @@ streams: ` }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -273,7 +274,7 @@ bucket_definitions: ` }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); const syncRulesContent = syncRules.syncConfigContent[0]; @@ -451,7 +452,7 @@ bucket_definitions: ` }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -585,7 +586,7 @@ bucket_definitions: ` }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); const syncRulesContent = syncRules.syncConfigContent[0]; @@ -761,7 +762,7 @@ bucket_definitions: const syncRules = await updateSyncRules(f, { content: BASIC_SYNC_RULES }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -824,7 +825,7 @@ bucket_definitions: content: BASIC_SYNC_RULES }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -869,7 +870,7 @@ bucket_definitions: content: BASIC_SYNC_RULES }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); const stream = sync.streamResponse({ syncContext, @@ -896,7 +897,7 @@ bucket_definitions: content: BASIC_SYNC_RULES }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); // Activate @@ -964,7 +965,7 @@ bucket_definitions: ` }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); @@ -1030,7 +1031,7 @@ bucket_definitions: ` }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2); @@ -1105,7 +1106,7 @@ bucket_definitions: ` }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const usersTable = await test_utils.resolveTestTable(writer, 'users', ['id'], config, 1); const listsTable = await test_utils.resolveTestTable(writer, 'lists', ['id'], config, 2); @@ -1175,7 +1176,7 @@ bucket_definitions: content: BASIC_SYNC_RULES }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); // Activate await writer.markAllSnapshotDone('0/0'); @@ -1215,7 +1216,7 @@ bucket_definitions: content: BASIC_SYNC_RULES }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); @@ -1306,7 +1307,7 @@ bucket_definitions: content: BASIC_SYNC_RULES }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); const bucket = bucketRequest(syncRules.syncConfigContent[0], 'mybucket[]').bucket; @@ -1466,7 +1467,7 @@ bucket_definitions: ` }); - const bucketStorage = await f.getInstance(syncRules); + const bucketStorage = await await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config); const highPriorityBucket = bucketRequest(syncRules.syncConfigContent[0], 'high_priority[]').bucket; @@ -1593,7 +1594,7 @@ bucket_definitions: content: BASIC_SYNC_RULES }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); await writer.markAllSnapshotDone('0/1'); @@ -1679,7 +1680,7 @@ config: const syncRules = await updateSyncRules(f, { content: rules[i] }); - const bucketStorage = f.getInstance(syncRules); + const bucketStorage = await getTestStorage(f, syncRules); await using writer = await bucketStorage.createWriter(test_utils.BATCH_OPTIONS); const testTable = await test_utils.resolveTestTable(writer, 'test', ['id'], config, i + 1); diff --git a/packages/service-core/src/replication/AbstractReplicator.ts b/packages/service-core/src/replication/AbstractReplicator.ts index 829c0a725..effcd3012 100644 --- a/packages/service-core/src/replication/AbstractReplicator.ts +++ b/packages/service-core/src/replication/AbstractReplicator.ts @@ -290,7 +290,7 @@ export abstract class AbstractReplicator; - abstract [Symbol.asyncDispose](): PromiseLike; + async [Symbol.asyncDispose](): Promise { + await this.iterateAsyncListeners(async (listener) => listener.beforeDispose?.()); + } } export interface BucketStorageFactoryListener { + /** Release owned resources before the factory closes its database connections. */ + beforeDispose: () => Promise; syncStorageCreated: (storage: SyncRulesBucketStorage) => void; replicationEvent: (event: ReplicationEventPayload) => void; } @@ -258,6 +262,11 @@ export function updateSyncRulesFromConfig( } export interface GetIntanceOptions { + /** + * The job lease, including a lease acquired during initial configuration. + * Required for writing; optional for reading. + */ + replicationLock?: ReplicationLock; /** * Set to true to skip trigger any events for creating the instance. *