diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8d16c5441..b4fe40bc8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -257,6 +257,9 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile --ignore-scripts + - name: Verify pinned mutation partition inventory + run: node --test scripts/mutation-partitions-engine.integration.mjs + - name: Rebuild the audited workspace build tool run: pnpm rebuild esbuild @@ -498,7 +501,9 @@ jobs: runs-on: ubuntu-latest # The governed air-gap codec target currently instruments 352 mutants and # legitimately exceeds 20 minutes on a hosted runner. - timeout-minutes: ${{ contains(fromJSON('["revenue-lineage-package","revenue-lineage-graph","sdk-revenue-listing-funding","output-lookup-session-records","output-lookup-session-payloads","wallet-recovery-codec","wallet-recovery-installation","wallet-recovery-store","wallet-funding-store","wallet-recovery-transitions","wallet-recovery-controller","root-eviction-storage","root-eviction-journal","root-eviction-records","wallet-retained-snapshot","wallet-snapshot-sync","wallet-snapshot-sync-destination","wallet-snapshot-sync-rows","wallet-snapshot-archive","wallet-snapshot-remote-http"]'), matrix.target) && 90 || 45 }} + # Complete snapshot source-guard service qualification took 58m35s locally, + # before hosted checkout/install. Preserve the 45-minute default elsewhere. + timeout-minutes: ${{ contains(fromJSON('["revenue-lineage-package","revenue-lineage-graph","sdk-revenue-listing-funding","output-lookup-session-records","output-lookup-session-payloads","wallet-recovery-codec","wallet-recovery-installation","wallet-recovery-store","wallet-funding-store","wallet-recovery-transitions","wallet-recovery-controller","root-eviction-storage","root-eviction-journal","root-eviction-records","wallet-retained-snapshot","wallet-snapshot-journal","wallet-snapshot-sync","wallet-snapshot-sync-destination","wallet-snapshot-sync-rows","wallet-snapshot-archive","wallet-snapshot-remote-http","wallet-snapshot-remote-reader","wallet-snapshot-remote-service"]'), matrix.target) && 90 || 45 }} permissions: contents: read strategy: @@ -602,6 +607,36 @@ jobs: with: pattern: mutation-wallet-retained-snapshot-* path: .mutation-parts/wallet-retained-snapshot + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + if: contains(fromJSON(needs.prepare.outputs.partition-targets || '[]'), 'wallet-snapshot-sync') + with: + pattern: mutation-wallet-snapshot-sync-* + path: .mutation-parts/wallet-snapshot-sync + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + if: contains(fromJSON(needs.prepare.outputs.partition-targets || '[]'), 'wallet-snapshot-journal') + with: + pattern: mutation-wallet-snapshot-journal-* + path: .mutation-parts/wallet-snapshot-journal + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + if: contains(fromJSON(needs.prepare.outputs.partition-targets || '[]'), 'wallet-snapshot-remote-http') + with: + pattern: mutation-wallet-snapshot-remote-http-* + path: .mutation-parts/wallet-snapshot-remote-http + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + if: contains(fromJSON(needs.prepare.outputs.partition-targets || '[]'), 'wallet-snapshot-remote-service') + with: + pattern: mutation-wallet-snapshot-remote-service-* + path: .mutation-parts/wallet-snapshot-remote-service + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + if: contains(fromJSON(needs.prepare.outputs.partition-targets || '[]'), 'wallet-snapshot-archive') + with: + pattern: mutation-wallet-snapshot-archive-* + path: .mutation-parts/wallet-snapshot-archive + - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + if: contains(fromJSON(needs.prepare.outputs.partition-targets || '[]'), 'wallet-snapshot-remote-reader') + with: + pattern: mutation-wallet-snapshot-remote-reader-* + path: .mutation-parts/wallet-snapshot-remote-reader - name: Require every selected canonical partition target gate if: needs.prepare.outputs.partition-targets != '[]' && needs.prepare.outputs.partition-targets != '' env: @@ -998,6 +1033,36 @@ jobs: name: build-outputs path: .ci-artifacts - run: tar --extract --gzip --file .ci-artifacts/build-outputs.tar.gz + - name: Verify native snapshot crash recovery + if: matrix.id == 'shard-1' + working-directory: packages/wallet/wallet-toolbox + run: node test/storage/snapshotArchiveCrash.cjs + - name: Provision pinned native snapshot MySQL fixture + if: matrix.id == 'shard-1' + working-directory: packages/wallet/wallet-toolbox + timeout-minutes: 5 + env: + TS_STACK_SNAPSHOT_HOSTED_MYSQL: '1' + run: node test/storage/snapshotArchiveDocker.cjs provision-hosted-image + - name: Verify native snapshot MySQL recovery + if: matrix.id == 'shard-1' + working-directory: packages/wallet/wallet-toolbox + timeout-minutes: 5 + env: + TS_STACK_SNAPSHOT_HOSTED_MYSQL: '1' + run: node test/storage/runSnapshotArchiveMysql.cjs + - name: Verify native journal SQLite recovery + if: matrix.id == 'shard-1' + working-directory: packages/wallet/wallet-toolbox + timeout-minutes: 3 + run: node test/storage/snapshotJournalSqliteCrash.cjs + - name: Verify native journal MySQL recovery + if: matrix.id == 'shard-1' + working-directory: packages/wallet/wallet-toolbox + timeout-minutes: 10 + env: + TS_STACK_SNAPSHOT_HOSTED_MYSQL: '1' + run: node test/storage/runSnapshotJournalMysql.cjs - name: Generate wallet-toolbox coverage shard env: WALLET_SHARD: ${{ matrix.shard }} diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 94cf6f84f..609d64396 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -32,10 +32,9 @@ jobs: language: - actions - javascript-typescript - # The only Python files are deterministic OpenAPI output under - # conformance/generated/**. The CodeQL config excludes that owned - # generated boundary, while codegen CI verifies its specs, locked - # generator, and output. + # Scan the independently authored BRC-118 conformance oracle. + # Generated OpenAPI clients remain excluded by the owned registry. + - python steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 diff --git a/.github/workflows/conformance.yml b/.github/workflows/conformance.yml index eae57f5bd..e94b87def 100644 --- a/.github/workflows/conformance.yml +++ b/.github/workflows/conformance.yml @@ -31,6 +31,8 @@ jobs: cache: pnpm - name: Install deps run: pnpm install --frozen-lockfile --ignore-scripts + - name: Verify independent BRC-118 wire and signature vectors + run: python3 conformance/runner/scripts/brc118-vectors.py - name: Build TS runner dependencies run: >- pnpm -r diff --git a/.github/workflows/mutation-tests.yml b/.github/workflows/mutation-tests.yml index 432040fbb..cd4d452c6 100644 --- a/.github/workflows/mutation-tests.yml +++ b/.github/workflows/mutation-tests.yml @@ -103,7 +103,9 @@ jobs: needs: prepare runs-on: ubuntu-latest # Preserve the acknowledged wallet/overlays90-minute target union. - timeout-minutes: ${{ contains(fromJSON('["revenue-lineage-package","revenue-lineage-graph","sdk-revenue-listing-funding","output-lookup-session-records","output-lookup-session-payloads","wallet-recovery-codec","wallet-recovery-installation","wallet-recovery-store","wallet-funding-store","wallet-recovery-transitions","wallet-recovery-controller","root-eviction-storage","root-eviction-journal","root-eviction-records","wallet-retained-snapshot","wallet-snapshot-sync","wallet-snapshot-sync-destination","wallet-snapshot-sync-rows","wallet-snapshot-archive","wallet-snapshot-remote-http"]'), matrix.target) && 90 || 45 }} + # Complete snapshot source-guard service qualification took 58m35s locally, + # before hosted checkout/install. Preserve the 45-minute default elsewhere. + timeout-minutes: ${{ contains(fromJSON('["revenue-lineage-package","revenue-lineage-graph","sdk-revenue-listing-funding","output-lookup-session-records","output-lookup-session-payloads","wallet-recovery-codec","wallet-recovery-installation","wallet-recovery-store","wallet-funding-store","wallet-recovery-transitions","wallet-recovery-controller","root-eviction-storage","root-eviction-journal","root-eviction-records","wallet-retained-snapshot","wallet-snapshot-journal","wallet-snapshot-sync","wallet-snapshot-sync-destination","wallet-snapshot-sync-rows","wallet-snapshot-archive","wallet-snapshot-remote-http","wallet-snapshot-remote-reader","wallet-snapshot-remote-service"]'), matrix.target) && 90 || 45 }} permissions: contents: read strategy: diff --git a/.sonarcloud.properties b/.sonarcloud.properties index 4743f8d61..8afe3f586 100644 --- a/.sonarcloud.properties +++ b/.sonarcloud.properties @@ -10,7 +10,7 @@ sonar.exclusions=packages/verifast/src/wasm/bdk-core.*,conformance/generated/**, # into self-contained Docker/package build contexts and checked byte-for-byte # in CI. Analyze the code for issues, but do not report intentional generated # copies as source duplication. -sonar.cpd.exclusions=**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.man.test.ts,**/__test__/**,**/__tests__/**,**/test/**,**/tests/**,**/*.vectors.ts,**/eslint.config.js,infra/wab/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/rateLimitPolicy.ts,infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts,infra/message-box-server/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/edgePolicy.ts,infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts,infra/message-box-server/src/security/edgePolicy.ts,infra/chaintracks-server/src/security/edgePolicy.ts,packages/overlays/overlay-express/src/security/edgePolicy.ts,packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts,infra/uhrp-server-cloud-bucket/src/resourceLimits.ts,infra/uhrp-server-cloud-bucket/src/utils/network.ts,infra/wallet-infra/src/KnexPaymentReplayStore.ts,infra/uhrp-server-basic/src/chirp/core/**,infra/uhrp-server-cloud-bucket/src/chirp/core/**,infra/uhrp-server-basic/src/chirp/openapi.ts,infra/uhrp-server-cloud-bucket/src/chirp/openapi.ts,infra/uhrp-server-cloud-bucket/src/chirp/contracts.ts,infra/uhrp-server-cloud-bucket/src/chirp/commitIndex.ts,infra/uhrp-server-cloud-bucket/src/chirp/routes.ts,infra/uhrp-server-cloud-bucket/src/chirp/bodyMiddleware.ts,infra/chaintracks-server/src/telemetry.ts,infra/wab/src/telemetry.ts,infra/uhrp-server-basic/src/telemetry.ts,infra/uhrp-server-cloud-bucket/src/telemetry.ts,infra/wallet-infra/src/telemetry.ts,infra/message-box-server/src/telemetry.ts +sonar.cpd.exclusions=**/*.test.ts,**/*.test.tsx,**/*.spec.ts,**/*.spec.tsx,**/*.man.test.ts,**/__test__/**,**/__tests__/**,**/test/**,**/tests/**,**/*.vectors.ts,**/eslint.config.js,infra/wab/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/rateLimitPolicy.ts,infra/uhrp-server-cloud-bucket/src/security/rateLimitPolicy.ts,infra/message-box-server/src/security/rateLimitPolicy.ts,infra/uhrp-server-basic/src/security/edgePolicy.ts,infra/uhrp-server-cloud-bucket/src/security/edgePolicy.ts,infra/message-box-server/src/security/edgePolicy.ts,infra/chaintracks-server/src/security/edgePolicy.ts,packages/overlays/overlay-express/src/security/edgePolicy.ts,packages/wallet/wallet-toolbox/src/storage/remoting/edgePolicy.ts,infra/uhrp-server-cloud-bucket/src/resourceLimits.ts,infra/uhrp-server-cloud-bucket/src/utils/network.ts,infra/wallet-infra/src/KnexPaymentReplayStore.ts,infra/uhrp-server-basic/src/chirp/core/**,infra/uhrp-server-cloud-bucket/src/chirp/core/**,infra/uhrp-server-basic/src/chirp/openapi.ts,infra/uhrp-server-cloud-bucket/src/chirp/openapi.ts,infra/uhrp-server-cloud-bucket/src/chirp/contracts.ts,infra/uhrp-server-cloud-bucket/src/chirp/commitIndex.ts,infra/uhrp-server-cloud-bucket/src/chirp/routes.ts,infra/uhrp-server-cloud-bucket/src/chirp/bodyMiddleware.ts,infra/chaintracks-server/src/telemetry.ts,infra/wab/src/telemetry.ts,infra/uhrp-server-basic/src/telemetry.ts,infra/uhrp-server-cloud-bucket/src/telemetry.ts,infra/wallet-infra/src/telemetry.ts,infra/message-box-server/src/telemetry.ts,infra/uhrp-server-cloud-bucket/dev/bin/nodemon.js,infra/uhrp-server-cloud-bucket/dev/check-watch.cjs,infra/wab/dev/bin/nodemon.js,infra/wab/dev/check-watch.cjs # Narrow compatibility exceptions are registered with owner, evidence, review # dates, and objective removal conditions in repository-health/exceptions.json. sonar.issue.ignore.multicriteria=werrProtocolNames,curveSingletonAlias,curveSingletonReturn,scriptOpcodeDispatch diff --git a/conformance/META.json b/conformance/META.json index 06dbbfcb9..3c8caf14a 100644 --- a/conformance/META.json +++ b/conformance/META.json @@ -20,10 +20,20 @@ "sdk.keys.publickey", "sdk.crypto.signature" ], - "BRC-74": ["sdk.transactions.merklepath", "broadcast.merklepath"], - "BRC-77": ["sdk.compat.bsm"], - "BRC-31": ["messaging.brc31.authrite-signature"], - "BRC-29": ["wallet.brc29.payment-derivation", "payments.brc29-payment-protocol"], + "BRC-74": [ + "sdk.transactions.merklepath", + "broadcast.merklepath" + ], + "BRC-77": [ + "sdk.compat.bsm" + ], + "BRC-31": [ + "messaging.brc31.authrite-signature" + ], + "BRC-29": [ + "wallet.brc29.payment-derivation", + "payments.brc29-payment-protocol" + ], "BRC-100": [ "wallet.brc100.getpublickey", "wallet.brc100.createhmac", @@ -55,26 +65,66 @@ "wallet.brc100.getversion", "wallet.storage.adapterconformance" ], - "BRC-121": ["payments.brc121"], - "BRC-26": ["storage.uhrp-http"], - "BRC-167": ["storage.chirp-v1"], - "BRC-62": ["overlay.submit"], - "BRC-22": ["overlay.lookup", "overlay.topicmanagement"], - "BRC-20": ["broadcast.arcsubmit", "broadcast.merklepath"], - "BRC-21": ["sync.gasprotocol"], - "BRC-40": ["sync.brc40"], - "BRC-14": ["sdk.scripts.evaluation"], - "merkle-service": ["broadcast.merkle-service"], - "message-box": ["messaging.messagebox-http"], - "chaintracks-v2": ["sync.chaintracks-v2-http"], - "BRC-141": ["transport.air-gap-optical"], - "BRC-103": ["auth.brc31-handshake", "messaging.authsocket"], - "BRC-104": ["auth.brc31-handshake"] + "BRC-121": [ + "payments.brc121" + ], + "BRC-26": [ + "storage.uhrp-http" + ], + "BRC-167": [ + "storage.chirp-v1" + ], + "BRC-62": [ + "overlay.submit" + ], + "BRC-22": [ + "overlay.lookup", + "overlay.topicmanagement" + ], + "BRC-20": [ + "broadcast.arcsubmit", + "broadcast.merklepath" + ], + "BRC-21": [ + "sync.gasprotocol" + ], + "BRC-40": [ + "sync.brc40" + ], + "BRC-14": [ + "sdk.scripts.evaluation" + ], + "merkle-service": [ + "broadcast.merkle-service" + ], + "message-box": [ + "messaging.messagebox-http" + ], + "chaintracks-v2": [ + "sync.chaintracks-v2-http" + ], + "BRC-141": [ + "transport.air-gap-optical" + ], + "BRC-103": [ + "auth.brc31-handshake", + "messaging.authsocket" + ], + "BRC-104": [ + "auth.brc31-handshake", + "payments.brc118" + ], + "BRC-118": [ + "payments.brc118" + ], + "BRC-105": [ + "payments.brc118" + ] }, "stats": { - "total_files": 77, - "total_vectors": 6700, - "last_updated": "2026-09-24" + "total_files": 78, + "total_vectors": 6706, + "last_updated": "2026-09-28" }, "regression_index": { "beef-v2-txid-panic": "go-sdk#306", diff --git a/conformance/PARITY_MATRIX.json b/conformance/PARITY_MATRIX.json index fd9a6648b..7b05e4cdf 100644 --- a/conformance/PARITY_MATRIX.json +++ b/conformance/PARITY_MATRIX.json @@ -1,21 +1,21 @@ { "schema_version": "1.0", - "generated_at": "2026-09-24", + "generated_at": "2026-09-28", "source": "ts-stack conformance corpus", "description": "Machine-readable parity status for cross-language SDK implementations (Go, Rust, Python). Use this to track and drive conformance.", "summary": { - "total_files": 77, - "total_vectors": 6700, - "fully_required_files": 58, + "total_files": 78, + "total_vectors": 6706, + "fully_required_files": 59, "files_with_intended": 17, "files_with_mixed_status": 15, "vectors_by_status": { - "required": 6496, + "required": 6502, "intended": 204, "skipped": 7 }, "by_reason_category": { - "fully_supported": 1283, + "fully_supported": 1289, "governed_vector_skip": 50, "historical_regression": 36, "partial_ts_behavioral_difference": 5116, @@ -145,6 +145,18 @@ "reason_category": "fully_supported", "categories": [] }, + { + "path": "payments/brc118.json", + "id": "payments.brc118", + "total_vectors": 6, + "file_level_parity": "required", + "effective_status": "required", + "required_count": 6, + "intended_count": 0, + "skipped_count": 0, + "reason_category": "fully_supported", + "categories": [] + }, { "path": "payments/brc121.json", "id": "payments.brc121", diff --git a/conformance/runner/scripts/brc118-vectors.py b/conformance/runner/scripts/brc118-vectors.py new file mode 100644 index 000000000..6f0b04ff6 --- /dev/null +++ b/conformance/runner/scripts/brc118-vectors.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python3 +"""Independent stdlib RFC 7578 reader, wire writer, and BRC-104 preimage oracle. + +Run without arguments to verify the checked-in corpus. --write regenerates it. +This is an independent fixture oracle, not a claimed external wallet integration. +""" +import base64 +import json +import struct +import sys +from email import policy +from email.parser import BytesParser +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[3] +DESTINATION = ROOT / 'conformance/vectors/payments/brc118.json' +REVISION = '2b959b13f1f73040d13cc4eb14edbfc376f8010b' +BOUNDARY = '----BsvPayment00112233445566778899aabbccddeeff' +PAYMENT = '{"derivationPrefix":"AA==","derivationSuffix":"AQ==","transaction":"AQID"}' + + +def varint(number): + if number < 0: + return b'\xff' + struct.pack(', + expected: Record +): Promise { + const payload = + input.payload_base64 === null + ? undefined + : { + bytes: new Uint8Array(Buffer.from(input.payload_base64 as string, 'base64')), + contentType: input.payload_content_type as string + } + const encoded = buildMultipartPayment( + input.payment_json as string, + payload, + 1024 * 1024, + input.boundary as string + ) + expect(Buffer.from(encoded.body).toString('base64')).toBe(expected.body_base64) + const client = new AuthFetch({} as WalletInterface) as unknown as { + serializeRequest( + method: string, + headers: Record, + body: Uint8Array, + url: URL, + nonce: number[] + ): Promise + } + const writer = await client.serializeRequest( + input.method as string, + input.headers as Record, + encoded.body, + new URL(`https://fixture.invalid${input.path as string}${input.query as string}`), + Array.from(Buffer.from(input.request_id_hex as string, 'hex')) + ) + expect(toHex(writer.toArray())).toBe(expected.request_preimage_hex) +} // ── Helpers ─────────────────────────────────────────────────────────────────── @@ -336,7 +376,7 @@ function dispatchBRC121(input: Record, expected: Record = ['brc29-payment-protocol', 'brc121'] +export const categories: ReadonlyArray = ['brc29-payment-protocol', 'brc121', 'brc118'] export function dispatch( category: string, @@ -344,6 +384,8 @@ export function dispatch( expected: Record ): void | Promise { switch (category) { + case 'brc118': + return dispatchBRC118(input, expected) case 'brc29-payment-protocol': return dispatchBRC29PaymentProtocol(input, expected) case 'brc121': diff --git a/conformance/vectors/payments/brc118.json b/conformance/vectors/payments/brc118.json new file mode 100644 index 000000000..db38431fb --- /dev/null +++ b/conformance/vectors/payments/brc118.json @@ -0,0 +1,168 @@ +{ + "$schema": "../../schema/vector.schema.json", + "id": "payments.brc118", + "name": "BRC-118 Multipart Transport and BRC-104 Request Preimages", + "version": "1.0.0", + "brc": ["BRC-104", "BRC-105", "BRC-118"], + "reference_impl": "@bsv/sdk; independent Python stdlib oracle", + "parity_class": "required", + "notes": "Reviewed BRCs commit 2b959b13f1f73040d13cc4eb14edbfc376f8010b; peer-to-peer/0104.md, payments/0105.md, payments/0118.md. Payment fields are synthetic transport bytes, not a spendable transaction. Non-multipart released signature preimages remain unchanged.", + "vectors": [ + { + "id": "payments.brc118.json-whitespace-utf8", + "description": "Exact multipart bytes and authenticated preimage: json-whitespace-utf8", + "input": { + "payment_json": "{\"derivationPrefix\":\"AA==\",\"derivationSuffix\":\"AQ==\",\"transaction\":\"AQID\"}", + "boundary": "----BsvPayment00112233445566778899aabbccddeeff", + "quoted_boundary": false, + "payload_base64": "eyAic25vdyI6ICLpm6oiIH0K", + "payload_content_type": "application/json; charset=utf-8", + "method": "POST", + "path": "/paid", + "query": "?q=%E9%9B%AA&order=1", + "request_id_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "headers": { + "authorization": "Bearer conformance-fixture", + "content-type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "x-bsv-example": "fixture" + } + }, + "expected": { + "body_base64": "LS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IngtYnN2LXBheW1lbnQiDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24NCg0KeyJkZXJpdmF0aW9uUHJlZml4IjoiQUE9PSIsImRlcml2YXRpb25TdWZmaXgiOiJBUT09IiwidHJhbnNhY3Rpb24iOiJBUUlEIn0NCi0tLS0tLUJzdlBheW1lbnQwMDExMjIzMzQ0NTU2Njc3ODg5OWFhYmJjY2RkZWVmZg0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJib2R5Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9qc29uOyBjaGFyc2V0PXV0Zi04DQoNCnsgInNub3ciOiAi6ZuqIiB9Cg0KLS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmLS0NCg==", + "content_type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "request_preimage_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f04504f5354052f70616964143f713d254539253942254141266f726465723d31030d617574686f72697a6174696f6e1a42656172657220636f6e666f726d616e63652d666978747572650c636f6e74656e742d747970654c6d756c7469706172742f666f726d2d646174613b20626f756e646172793d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d782d6273762d6578616d706c650766697874757265fdae012d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22782d6273762d7061796d656e74220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6a736f6e0d0a0d0a7b2264657269766174696f6e507265666978223a2241413d3d222c2264657269766174696f6e537566666978223a2241513d3d222c227472616e73616374696f6e223a2241514944227d0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22626f6479220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6a736f6e3b20636861727365743d7574662d380d0a0d0a7b2022736e6f77223a2022e99baa22207d0a0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566662d2d0d0a" + }, + "tags": ["brc-118", "exact-bytes", "independent-python-oracle"] + }, + { + "id": "payments.brc118.binary-nul", + "description": "Exact multipart bytes and authenticated preimage: binary-nul", + "input": { + "payment_json": "{\"derivationPrefix\":\"AA==\",\"derivationSuffix\":\"AQ==\",\"transaction\":\"AQID\"}", + "boundary": "----BsvPayment00112233445566778899aabbccddeeff", + "quoted_boundary": false, + "payload_base64": "AID/DQoA", + "payload_content_type": "application/octet-stream", + "method": "POST", + "path": "/paid", + "query": "?q=%E9%9B%AA&order=1", + "request_id_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "headers": { + "authorization": "Bearer conformance-fixture", + "content-type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "x-bsv-example": "fixture" + } + }, + "expected": { + "body_base64": "LS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IngtYnN2LXBheW1lbnQiDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24NCg0KeyJkZXJpdmF0aW9uUHJlZml4IjoiQUE9PSIsImRlcml2YXRpb25TdWZmaXgiOiJBUT09IiwidHJhbnNhY3Rpb24iOiJBUUlEIn0NCi0tLS0tLUJzdlBheW1lbnQwMDExMjIzMzQ0NTU2Njc3ODg5OWFhYmJjY2RkZWVmZg0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJib2R5Ig0KQ29udGVudC1UeXBlOiBhcHBsaWNhdGlvbi9vY3RldC1zdHJlYW0NCg0KAID/DQoADQotLS0tLS1Cc3ZQYXltZW50MDAxMTIyMzM0NDU1NjY3Nzg4OTlhYWJiY2NkZGVlZmYtLQ0K", + "content_type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "request_preimage_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f04504f5354052f70616964143f713d254539253942254141266f726465723d31030d617574686f72697a6174696f6e1a42656172657220636f6e666f726d616e63652d666978747572650c636f6e74656e742d747970654c6d756c7469706172742f666f726d2d646174613b20626f756e646172793d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d782d6273762d6578616d706c650766697874757265fd9b012d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22782d6273762d7061796d656e74220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6a736f6e0d0a0d0a7b2264657269766174696f6e507265666978223a2241413d3d222c2264657269766174696f6e537566666978223a2241513d3d222c227472616e73616374696f6e223a2241514944227d0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22626f6479220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6f637465742d73747265616d0d0a0d0a0080ff0d0a000d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566662d2d0d0a" + }, + "tags": ["brc-118", "exact-bytes", "independent-python-oracle"] + }, + { + "id": "payments.brc118.empty", + "description": "Exact multipart bytes and authenticated preimage: empty", + "input": { + "payment_json": "{\"derivationPrefix\":\"AA==\",\"derivationSuffix\":\"AQ==\",\"transaction\":\"AQID\"}", + "boundary": "----BsvPayment00112233445566778899aabbccddeeff", + "quoted_boundary": false, + "payload_base64": "", + "payload_content_type": "text/plain", + "method": "POST", + "path": "/paid", + "query": "?q=%E9%9B%AA&order=1", + "request_id_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "headers": { + "authorization": "Bearer conformance-fixture", + "content-type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "x-bsv-example": "fixture" + } + }, + "expected": { + "body_base64": "LS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IngtYnN2LXBheW1lbnQiDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24NCg0KeyJkZXJpdmF0aW9uUHJlZml4IjoiQUE9PSIsImRlcml2YXRpb25TdWZmaXgiOiJBUT09IiwidHJhbnNhY3Rpb24iOiJBUUlEIn0NCi0tLS0tLUJzdlBheW1lbnQwMDExMjIzMzQ0NTU2Njc3ODg5OWFhYmJjY2RkZWVmZg0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJib2R5Ig0KQ29udGVudC1UeXBlOiB0ZXh0L3BsYWluDQoNCg0KLS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmLS0NCg==", + "content_type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "request_preimage_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f04504f5354052f70616964143f713d254539253942254141266f726465723d31030d617574686f72697a6174696f6e1a42656172657220636f6e666f726d616e63652d666978747572650c636f6e74656e742d747970654c6d756c7469706172742f666f726d2d646174613b20626f756e646172793d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d782d6273762d6578616d706c650766697874757265fd87012d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22782d6273762d7061796d656e74220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6a736f6e0d0a0d0a7b2264657269766174696f6e507265666978223a2241413d3d222c2264657269766174696f6e537566666978223a2241513d3d222c227472616e73616374696f6e223a2241514944227d0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22626f6479220d0a436f6e74656e742d547970653a20746578742f706c61696e0d0a0d0a0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566662d2d0d0a" + }, + "tags": ["brc-118", "exact-bytes", "independent-python-oracle"] + }, + { + "id": "payments.brc118.absent", + "description": "Exact multipart bytes and authenticated preimage: absent", + "input": { + "payment_json": "{\"derivationPrefix\":\"AA==\",\"derivationSuffix\":\"AQ==\",\"transaction\":\"AQID\"}", + "boundary": "----BsvPayment00112233445566778899aabbccddeeff", + "quoted_boundary": false, + "payload_base64": null, + "payload_content_type": null, + "method": "POST", + "path": "/paid", + "query": "?q=%E9%9B%AA&order=1", + "request_id_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "headers": { + "authorization": "Bearer conformance-fixture", + "content-type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "x-bsv-example": "fixture" + } + }, + "expected": { + "body_base64": "LS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IngtYnN2LXBheW1lbnQiDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24NCg0KeyJkZXJpdmF0aW9uUHJlZml4IjoiQUE9PSIsImRlcml2YXRpb25TdWZmaXgiOiJBUT09IiwidHJhbnNhY3Rpb24iOiJBUUlEIn0NCi0tLS0tLUJzdlBheW1lbnQwMDExMjIzMzQ0NTU2Njc3ODg5OWFhYmJjY2RkZWVmZi0tDQo=", + "content_type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "request_preimage_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f04504f5354052f70616964143f713d254539253942254141266f726465723d31030d617574686f72697a6174696f6e1a42656172657220636f6e666f726d616e63652d666978747572650c636f6e74656e742d747970654c6d756c7469706172742f666f726d2d646174613b20626f756e646172793d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d782d6273762d6578616d706c650766697874757265fd0a012d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22782d6273762d7061796d656e74220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6a736f6e0d0a0d0a7b2264657269766174696f6e507265666978223a2241413d3d222c2264657269766174696f6e537566666978223a2241513d3d222c227472616e73616374696f6e223a2241514944227d0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566662d2d0d0a" + }, + "tags": ["brc-118", "exact-bytes", "independent-python-oracle"] + }, + { + "id": "payments.brc118.nested-multipart", + "description": "Exact multipart bytes and authenticated preimage: nested-multipart", + "input": { + "payment_json": "{\"derivationPrefix\":\"AA==\",\"derivationSuffix\":\"AQ==\",\"transaction\":\"AQID\"}", + "boundary": "----BsvPayment00112233445566778899aabbccddeeff", + "quoted_boundary": false, + "payload_base64": "LS1pbm5lcg0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJmaWVsZCINCg0KdmFsdWUNCi0taW5uZXItLQ0K", + "payload_content_type": "multipart/form-data; boundary=inner", + "method": "POST", + "path": "/paid", + "query": "?q=%E9%9B%AA&order=1", + "request_id_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "headers": { + "authorization": "Bearer conformance-fixture", + "content-type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "x-bsv-example": "fixture" + } + }, + "expected": { + "body_base64": "LS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IngtYnN2LXBheW1lbnQiDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24NCg0KeyJkZXJpdmF0aW9uUHJlZml4IjoiQUE9PSIsImRlcml2YXRpb25TdWZmaXgiOiJBUT09IiwidHJhbnNhY3Rpb24iOiJBUUlEIn0NCi0tLS0tLUJzdlBheW1lbnQwMDExMjIzMzQ0NTU2Njc3ODg5OWFhYmJjY2RkZWVmZg0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJib2R5Ig0KQ29udGVudC1UeXBlOiBtdWx0aXBhcnQvZm9ybS1kYXRhOyBib3VuZGFyeT1pbm5lcg0KDQotLWlubmVyDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9ImZpZWxkIg0KDQp2YWx1ZQ0KLS1pbm5lci0tDQoNCi0tLS0tLUJzdlBheW1lbnQwMDExMjIzMzQ0NTU2Njc3ODg5OWFhYmJjY2RkZWVmZi0tDQo=", + "content_type": "multipart/form-data; boundary=----BsvPayment00112233445566778899aabbccddeeff", + "request_preimage_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f04504f5354052f70616964143f713d254539253942254141266f726465723d31030d617574686f72697a6174696f6e1a42656172657220636f6e666f726d616e63652d666978747572650c636f6e74656e742d747970654c6d756c7469706172742f666f726d2d646174613b20626f756e646172793d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d782d6273762d6578616d706c650766697874757265fdeb012d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22782d6273762d7061796d656e74220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6a736f6e0d0a0d0a7b2264657269766174696f6e507265666978223a2241413d3d222c2264657269766174696f6e537566666978223a2241513d3d222c227472616e73616374696f6e223a2241514944227d0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22626f6479220d0a436f6e74656e742d547970653a206d756c7469706172742f666f726d2d646174613b20626f756e646172793d696e6e65720d0a0d0a2d2d696e6e65720d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d226669656c64220d0a0d0a76616c75650d0a2d2d696e6e65722d2d0d0a0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566662d2d0d0a" + }, + "tags": ["brc-118", "exact-bytes", "independent-python-oracle"] + }, + { + "id": "payments.brc118.quoted-boundary", + "description": "Exact multipart bytes and authenticated preimage: quoted-boundary", + "input": { + "payment_json": "{\"derivationPrefix\":\"AA==\",\"derivationSuffix\":\"AQ==\",\"transaction\":\"AQID\"}", + "boundary": "----BsvPayment00112233445566778899aabbccddeeff", + "quoted_boundary": true, + "payload_base64": "cXVvdGVk", + "payload_content_type": "text/plain", + "method": "POST", + "path": "/paid", + "query": "?q=%E9%9B%AA&order=1", + "request_id_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f", + "headers": { + "authorization": "Bearer conformance-fixture", + "content-type": "multipart/form-data; boundary=\"----BsvPayment00112233445566778899aabbccddeeff\"", + "x-bsv-example": "fixture" + } + }, + "expected": { + "body_base64": "LS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmDQpDb250ZW50LURpc3Bvc2l0aW9uOiBmb3JtLWRhdGE7IG5hbWU9IngtYnN2LXBheW1lbnQiDQpDb250ZW50LVR5cGU6IGFwcGxpY2F0aW9uL2pzb24NCg0KeyJkZXJpdmF0aW9uUHJlZml4IjoiQUE9PSIsImRlcml2YXRpb25TdWZmaXgiOiJBUT09IiwidHJhbnNhY3Rpb24iOiJBUUlEIn0NCi0tLS0tLUJzdlBheW1lbnQwMDExMjIzMzQ0NTU2Njc3ODg5OWFhYmJjY2RkZWVmZg0KQ29udGVudC1EaXNwb3NpdGlvbjogZm9ybS1kYXRhOyBuYW1lPSJib2R5Ig0KQ29udGVudC1UeXBlOiB0ZXh0L3BsYWluDQoNCnF1b3RlZA0KLS0tLS0tQnN2UGF5bWVudDAwMTEyMjMzNDQ1NTY2Nzc4ODk5YWFiYmNjZGRlZWZmLS0NCg==", + "content_type": "multipart/form-data; boundary=\"----BsvPayment00112233445566778899aabbccddeeff\"", + "request_preimage_hex": "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f04504f5354052f70616964143f713d254539253942254141266f726465723d31030d617574686f72697a6174696f6e1a42656172657220636f6e666f726d616e63652d666978747572650c636f6e74656e742d747970654e6d756c7469706172742f666f726d2d646174613b20626f756e646172793d222d2d2d2d4273765061796d656e743030313132323333343435353636373738383939616162626363646465656666220d782d6273762d6578616d706c650766697874757265fd8d012d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22782d6273762d7061796d656e74220d0a436f6e74656e742d547970653a206170706c69636174696f6e2f6a736f6e0d0a0d0a7b2264657269766174696f6e507265666978223a2241413d3d222c2264657269766174696f6e537566666978223a2241513d3d222c227472616e73616374696f6e223a2241514944227d0d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566660d0a436f6e74656e742d446973706f736974696f6e3a20666f726d2d646174613b206e616d653d22626f6479220d0a436f6e74656e742d547970653a20746578742f706c61696e0d0a0d0a71756f7465640d0a2d2d2d2d2d2d4273765061796d656e7430303131323233333434353536363737383839396161626263636464656566662d2d0d0a" + }, + "tags": ["brc-118", "exact-bytes", "independent-python-oracle"] + } + ] +} diff --git a/docs/conformance/contributing-vectors.md b/docs/conformance/contributing-vectors.md index 80cee3ae3..206b17b3b 100644 --- a/docs/conformance/contributing-vectors.md +++ b/docs/conformance/contributing-vectors.md @@ -84,7 +84,7 @@ conformance/vectors/ broadcast/*.json messaging/{authsocket,brc31/,message-box-http}.json overlay/{lookup,submit,topic-management}.json - payments/{brc121,brc29-payment-protocol}.json + payments/{brc118,brc121,brc29-payment-protocol}.json regressions/ (12 files — special format with regression.issue metadata) sdk/compat/bsm.json sdk/crypto/ (8 files: aes, ecdsa, ecies, hash160, hmac, ripemd160, sha256, signature) diff --git a/docs/conformance/vectors.md b/docs/conformance/vectors.md index e40b2b643..3285a3138 100644 --- a/docs/conformance/vectors.md +++ b/docs/conformance/vectors.md @@ -27,7 +27,7 @@ conformance/vectors/ broadcast/{arc-submit,merkle-path-validation,merkle-service}.json messaging/{authsocket,brc31/authrite-signature,message-box-http}.json overlay/{lookup,submit,topic-management}.json - payments/{brc121,brc29-payment-protocol}.json + payments/{brc118,brc121,brc29-payment-protocol}.json regressions/*.json (12 files) sdk/compat/bsm.json sdk/crypto/{aes,ecdsa,ecies,hash160,hmac,ripemd160,sha256,signature}.json diff --git a/docs/guides/brc118-payments.md b/docs/guides/brc118-payments.md new file mode 100644 index 000000000..70c5e830e --- /dev/null +++ b/docs/guides/brc118-payments.md @@ -0,0 +1,206 @@ +--- +id: guide-brc118-payments +title: 'BRC-118 Authenticated Multipart Payments' +kind: guide +version: '1.0.0' +last_updated: '2026-09-23' +last_verified: '2026-09-23' +review_cadence_days: 30 +status: beta +tags: [guide, payments, authentication, brc-105, brc-118] +--- + +# BRC-118 authenticated multipart payments + +BRC-118 carries a BRC-105 payment in an authenticated multipart request when +the payment BEEF would exceed a header budget. Small payments continue to use +`x-bsv-payment`. Deploy receiver support before enabling multipart clients. +The SDK, auth middleware and payment middleware must all support this profile. + +## Enable the receiver + +Install raw-byte authentication before any body parser. The middleware verifies +the complete transmitted body and the exact multipart Content-Type, including +its boundary, before extracting payment or application data. + +```ts +import express from 'express' +import { createAuthMiddleware } from '@bsv/auth-express-middleware' +import { createPaymentMiddleware } from '@bsv/payment-express-middleware' +import type { WalletInterface } from '@bsv/sdk' + +export function paidApp(wallet: WalletInterface) { + const app = express() + app.use(createAuthMiddleware({ wallet, captureRawBody: true })) + app.use( + createPaymentMiddleware({ + wallet, + enableMultipart: true, + calculateRequestPrice: () => 1 + }) + ) + app.post('/paid', (req, res) => res.json({ received: req.body })) + return app +} +``` + +`captureRawBody` and `enableMultipart` are opt-in. A verified 402 advertises +`x-bsv-payment-transports: header,multipart` only when both are active. With +either absent, the payment middleware advertises header support only. An older +server with no advertisement is also treated as header-only. + +The existing amount, BRC-29 derivation, BEEF validation, wallet internalization +and transaction-ID replay checks apply to either transport. Configure a shared, +durable atomic replay store across server processes; the default store is +bounded and process-local. A wallet exception retains the released 400 `ERR_PAYMENT_FAILED` response; a +replay-store failure returns 503. Neither proves that wallet work was rolled +back, and neither carries a new payment challenge. Reconcile the submitted +transaction before paying again. Payment transport is not durable result redelivery. + +## Preserve application bytes + +The wrapper contains one JSON `x-bsv-payment` part and at most one `body` part. +The latter holds the original bytes and media type. The route receives the +inner Content-Type and `req.rawBody`; `req.body` is decoded for JSON, text and +URL-encoded media types, and remains bytes for binary or nested multipart data. +JSON whitespace and UTF-8 bytes survive in `rawBody`. An empty part is distinct +from an absent part. The route never receives the outer payment in its body. + +Boundaries may be quoted or unquoted, but must be a single 1–70 character MIME +token. Framing uses CRLF. Duplicate parts or part headers, conflicting header +and body payments, unexpected paid-wrapper parts, unsupported part headers, +truncation and epilogues are rejected. An original multipart application payload +is an opaque inner body, not recursively parsed. Its initial unpaid request must +fit the receiver's bounded multipart profile (at most 128 parts, ASCII part +headers); applications needing a broader upload format should use a separate +upload route and pay for its resulting resource. + +Existing nonempty non-multipart signature preimages retain their released normalization. +Empty byte requests now consistently use BRC-104's `-1` sentinel in AuthFetch +and auth middleware, including the existing `express.raw` integration without +raw capture. SDK clients and auth receivers that previously signed/reconstructed +length `0` for empty byte arrays must adopt the integrated SDK 3.0.0 candidate and auth middleware 2.3.0 +together for those requests. This correction does not change nonempty bodies or +multipart payload-part bytes; an empty multipart payload part remains distinct +from a missing part. +Multipart authenticates the exact Content-Type value. There is no second, +weaker verification attempt with the boundary removed. + +## Prepare a payment before broadcasting + +`AuthFetch` snapshots the original request, creates a payment with +`noSend: true`, validates its actual Atomic BEEF/output and serializes the final +authenticated request before submitting it with `sendWith`. The wallet must +honor the BRC-100 prepare/submit contract and implement `abortAction`; wallets +that cannot do so must not use this automatic payment path. There is no fallback +to creating and immediately broadcasting a replacement transaction. + +```ts +import { AuthFetch, WalletClient } from '@bsv/sdk' + +const client = new AuthFetch(new WalletClient()) +const response = await client.fetch('https://example.com/paid', { + method: 'POST', + headers: { 'content-type': 'application/json; charset=utf-8' }, + body: '{ "message": "hello" }', + paymentTransport: { maxPaymentHeaderBytes: 4096 }, + signal: new AbortController().signal +}) +``` + +Small payments use a header when advertised. Larger payments require multipart +support. The original method, URL and query are preserved. Fetch forbids bodies +on GET/HEAD, so payments that require multipart on those methods fail before +broadcast; choose a server-supported POST route explicitly. Oversized +header-only attempts are refused, rather than relying on the standard's +recommended header fallback to deliver an already broadcast payment. + +Pass replayable, owned bytes for binary and original multipart bodies. Serialize +native FormData once with a Request, retain its generated Content-Type and read +its arrayBuffer before calling AuthFetch. Text-only FormData retains the legacy +URL-encoded behavior; file-bearing FormData and streams are rejected instead of +silently dropping files or changing signed bytes. + +Transport changes and bounded delivery retries reuse the same transaction. +A changed amount, derivation prefix or authenticated recipient stops automatic +payment and requires reconciliation. Prepared reservations are aborted on +refusal or cancellation where the wallet allows it. Once submission starts, a +lost acknowledgement is uncertain: it is not safe to abort or create another +payment automatically. Cancellation stops waiting and pending listeners; +already dispatched network or wallet work can still complete. + +`PaymentTransportError` has a permanent `code`, `retryable: false` and, when +available, a payment transaction ID and `prepared`, `submitted` or `uncertain` +state. A prepared refusal can additionally report `aborted`. HTTP 413/431 errors +are terminal and distinguish authenticated server replies from unsigned proxy +failures through `authenticated` and `httpStatus`. Reconcile uncertain outcomes +using the wallet and service before starting a new payment. No durable refund or +paid-result journal is implied. + +## Bound resource use and expose negotiation to browsers + +| Bound | Default | +| -------------------------------------------------- | ---------------------------------------------- | +| SDK payment header selection | 8,192 bytes | +| SDK aggregate request headers | 16 KiB, including a 4 KiB auth/framing reserve | +| SDK and payment middleware payment JSON | 4 MiB | +| SDK and payment middleware complete body | 7 MiB | +| Auth middleware complete request encoding | 8 MiB | +| Raw receiver retained bodies | 64 MiB aggregate per middleware instance | +| Raw receiver pending requests / collection timeout | 1,000 / 30 seconds | +| Multipart part headers | 2,048 bytes per part | + +These are bounds, not promises about an intermediary. Set lower client budgets +for a known proxy, and align proxy, auth and payment middleware limits. The raw +collector rejects compressed bodies, malformed lengths, premature disconnects, +slow input, exhausted capacity and oversized streams before authentication. +Body collection cannot be disabled or made unbounded in raw mode. Multipart +removes header pressure, not body limits. + +Keep the service's credential-free cross-origin access policy. Handle OPTIONS +before authentication, allow the request Content-Type and `x-bsv-auth-*`/ +`x-bsv-payment` headers, and expose the existing authentication response headers +plus `x-bsv-payment-transports`, `x-bsv-payment-version`, +`x-bsv-payment-satoshis-required`, `x-bsv-payment-derivation-prefix` and the optional +`x-bsv-payment-known-txids`. Multipart requires no credentialed browser mode or +new origin allowlist. Use explicit names in CORS libraries that do not support +header-prefix matching. + +## Compatibility and evidence + +The profile pins BRC-104/105/118 at BRCs revision +[`2b959b13f1f73040d13cc4eb14edbfc376f8010b`](https://github.com/bsv-blockchain/BRCs/tree/2b959b13f1f73040d13cc4eb14edbfc376f8010b). +Six shared wire/preimage fixtures under `conformance/vectors/payments/brc118.json` +are independently generated and read by Python's standard-library MIME parser. +The SDK writer, auth verifier and payment parser check the same fixtures. Run +`python3 conformance/runner/scripts/brc118-vectors.py` to verify the independent +oracle. Composed HTTP tests use two independent wallet identities, real mutual +authentication and enforced proxy header/body limits; they do not spend funds. + +External implementation availability is not an interoperability claim. The +reviewed [Python implementation](https://github.com/Calgooon/codex-x402/tree/2f9f89b7769bb05a3dc81c9e287e16f925773f5b) +has a multipart writer but strips Content-Type parameters in its signing path +and omits empty body parts. Its complete authenticated path therefore does not +match these boundary-binding vectors. No live Rust/Workers peer or external +owner sign-off is claimed. + +`@bsv/402-pay` implements the separate BRC-121 header protocol +(`x-bsv-beef`, sender, nonce, time and vout); it does not use BRC-105 AuthFetch +negotiation. Its [guide](./http-402-payments.md) and wire contract remain separate. +PeerPay MessageBox receive compatibility likewise does not increase a relay's +encrypted-envelope limit or make an already broadcast oversized message safe. + +The native Chromium gate exercises credential-free cross-origin preflights, +large binary and exact-JSON payments, and a header-only server. Hiding the signed +negotiation header through CORS invalidates the response signature before any +payment preparation; expose the complete signed response header set. + +Published SDK 2.8.0 was also tested in the same native-browser fixture against +both new receiver configurations (multipart enabled and disabled). Its small +header payment is accepted once with the exact application body. This uses the +registry tarball after SHA-512 integrity verification, independently from the +workspace SDK build. To repeat the additional historical-client probe, extract +that verified package and point `BRC118_LEGACY_SDK_MODULE` at its +`dist/esm/mod.js` when running the payment package's `test:browser`; the complete +current-client gate still runs first. This does not claim that a legacy client +can send payments above the legacy header limit. diff --git a/docs/guides/index.md b/docs/guides/index.md index 0bd8bf68b..406ba779d 100644 --- a/docs/guides/index.md +++ b/docs/guides/index.md @@ -69,7 +69,14 @@ creator, issuer, Payee, wallet, evidence, Delivery, and storage roles. **Time:** ~45 minutes | **Level:** Advanced -### 7. [Understand and Contribute Registry Metadata](./registry-metadata.md) +### 7. [BRC-118 Authenticated Multipart Payments](./brc118-payments.md) + +Carry larger BRC-105 payments with exact-byte authentication, bounded parsing, +receiver-first negotiation and preparation before broadcast. + +**Time:** ~25 minutes | **Level:** Advanced + +### 8. [Understand and Contribute Registry Metadata](./registry-metadata.md) Document protocols through the BRC process, request accurate optional descriptions, and keep registry stewardship separate from wallet permissions. Includes exact @@ -98,6 +105,12 @@ lookup identities, publisher choice, updates and wallet fallback expectations. **Looking for infrastructure examples?** Check [Infrastructure Components](../infrastructure/). +## Resumable wallet synchronization + +[Sync reliability and proof recovery](wallet-sync-reliability.md) describes the +Toolbox 2.14 candidate API, cancellation, atomic checkpoints, foreground fairness, +proof recovery, benchmark scope and the next coherent-snapshot milestone. + ## Identity, DIDs and credentials - [Unified integration and implementer guidance](identity-did-vc.md) diff --git a/docs/guides/wallet-backup-recovery.md b/docs/guides/wallet-backup-recovery.md index 0d89d0a2d..00598d7ef 100644 --- a/docs/guides/wallet-backup-recovery.md +++ b/docs/guides/wallet-backup-recovery.md @@ -3,8 +3,8 @@ id: wallet-backup-recovery title: 'Wallet Backup and Recovery' kind: guide version: '1.0.0' -last_updated: '2026-09-24' -last_verified: '2026-09-24' +last_updated: '2026-09-29' +last_verified: '2026-09-29' review_cadence_days: 30 status: stable tags: [wallet, backup, recovery, brc100, brc38, brc39] @@ -121,12 +121,33 @@ records. Storage operators should also use the ## Make recovery understandable in the product +For device-based backups, expose the available source devices and retained +generations, their recovery points, and whether each initial copy completed. +A newly registered empty device can have the newest timestamp. Do not treat +that timestamp, a matching identity or a zero balance as proof that it is the +right recovery source. Keep older known-good copies reachable, and provide a +supported way to select one when automatic recovery cannot establish coverage. +If the backup protocol combines histories from several devices, qualify that +replay and its conflict rules; do not improvise a database merge in the UI. + +Complete source selection and validate the recovered inventory before starting +the new installation's backup writers. Test two successive reinstalls, including +a newer empty or incomplete device alongside an older complete backup. A single +successful reinstall does not establish this behavior. These device/generation +controls belong to the wallet host's backup system; the BRC-39 file helpers do +not discover remote backups or choose a source device. + Present separate statuses for key recovery and wallet-data backup. Show the selected profile/network, last completed export or backup time, last successful restore test, and any records created since the recovery point. Do not mark a wallet “fully backed up” merely because a seed was displayed, a download was started, or a remote replica is reachable. +After a receive, show whether its transaction and derivation records are covered +by an acknowledged retained recovery point. An on-chain confirmation alone does +not establish that those records survived elsewhere. Keep pending or failed +backup status visible before users uninstall or replace a device. + Suggested wording to adapt to the product's verified behavior: > **Keep both parts.** Your recovery key restores control of this wallet. Your diff --git a/docs/guides/wallet-data-portability.md b/docs/guides/wallet-data-portability.md index a99b57dc3..7de935aee 100644 --- a/docs/guides/wallet-data-portability.md +++ b/docs/guides/wallet-data-portability.md @@ -3,8 +3,8 @@ id: wallet-data-portability title: 'BRC-38/39 Wallet Data Portability' kind: guide version: '1.0.0' -last_updated: '2026-09-24' -last_verified: '2026-09-24' +last_updated: '2026-09-30' +last_verified: '2026-09-30' review_cadence_days: 30 status: stable tags: [wallet, backup, interoperability, brc38, brc39] @@ -35,6 +35,43 @@ upgrading. Pending sync work, including [#569](https://github.com/bsv-blockchain/ts-stack/pull/569), is not an available API contract for this guide. +### Unpublished 2.15 source-capture checkpoint + +PR #569 now captures BRC-38 source metadata, user and all standard table reads +inside one provider read view. SQLite uses one transaction; MySQL explicitly +requests repeatable-read isolation; IndexedDB uses one readonly transaction +covering settings and the wallet stores. SQLite WAL permits an independent +writer during capture. IndexedDB queues overlapping writers until capture ends; +this checkpoint does not claim bounded foreground write latency for that phase. +A local MySQL 8.4.11 fixture with independent connections verifies source +isolation, enforced read-only access, unchanged session defaults and failure +cleanup. This is not deployed-provider or PXC recovery qualification. + +Pass `{ requireSnapshot: true }` to `exportBRC38`, `exportBRC38Json` or the +`exportBRC39` options to require a coherent source view. Custom `StorageProvider` +implementations opt in with `supportsReadSnapshot()` and `readSnapshot(callback)` +and must honor the token on every query. A required but unsupported snapshot +is refused before table capture. Without that option, old custom providers retain +the documented caller-quiesced legacy path; it does not gain a snapshot guarantee. +The callback is for database capture only, without peer I/O or progress handlers. +Source JSON history omits null/undefined values only for recognized optional +object properties in a detached copy, retaining false, zero, empty strings and +every array position. Required values remain subject to validation. A null array +entry is rejected rather than dropped. The source records are not rewritten. + +The candidate also adds a local SQL `openReadSnapshot` lifetime with explicit +close/cancellation/expiry and one read at a time. It retains a pinned transaction +across idle periods and occupies a pool connection until physical cleanup. +IndexedDB and remote clients do not gain this capability. The existing export +helpers continue to use their scoped capture path; see +[retained SQL view limits](wallet-sync-reliability.md#retained-local-sql-read-views-unpublished-candidate). + +This is an intermediate source candidate, not a released streaming export API. +The materialized helpers below still allocate the full document/file. Remote +snapshot handles, bounded immutable paging, streaming files, staged recovery and +the push/backup scheduling changes remain required parts of the +[full implementation program](https://github.com/bsv-blockchain/ts-stack/blob/codex/wallet-sync-interop-reliability/specs/wallet/sync-portability-program.md). + ## Coverage and limits The implementation exports one `user`, its `sourceStorage` metadata and 13 @@ -50,7 +87,23 @@ profiles, storage-global monitor events, or product data held outside these tables. Inventory contacts, permissions, external files and custom signing dependencies in your product before describing its backup coverage. -The current helpers read multiple tables and materialize the complete document +This exclusion also applies to application-owned tables in the **same database**: +reports, sessions and other custom tables are not discovered or copied by the +portable helpers. Preserving an output does not preserve its application meaning +when that meaning exists only in a custom table. Use standard baskets, tags, +labels and their relationships where their semantics fit; otherwise provide a +separately versioned application export/import and test its links to restored +outpoints. Distinguish durable user content from disposable sessions, caches and +credentials, which must not be copied indiscriminately. A successful wallet +import must disclose any missing application context. + +For an older source wallet, check its installed exports and schema before +planning a drill. If the required helpers are absent, preserve the original and +qualify an isolated copy through the supported migration/export path. Record +both versions; importing a file on the candidate does not prove that an older +wallet can produce that file unchanged. + +The released helpers read multiple tables and materialize the complete document and encrypted file in memory. They do not take a database-wide snapshot, provide streaming archive I/O, expose an archive progress/cancellation API, or promise a coherent view while other writers change the source. Use a stable diff --git a/docs/guides/wallet-recovery-agent-brief.md b/docs/guides/wallet-recovery-agent-brief.md index f8368ad9d..123326658 100644 --- a/docs/guides/wallet-recovery-agent-brief.md +++ b/docs/guides/wallet-recovery-agent-brief.md @@ -3,8 +3,8 @@ id: wallet-recovery-agent-brief title: 'Wallet Recovery Implementation Brief for AI Agents' kind: guide version: '1.0.0' -last_updated: '2026-09-24' -last_verified: '2026-09-24' +last_updated: '2026-09-29' +last_verified: '2026-09-29' review_cadence_days: 30 status: stable tags: [wallet, recovery, backup, agents, implementation] @@ -57,6 +57,12 @@ Provide encrypted export for the selected profile with a recoverable passphrase, confirmed file-save completion, freshness information and clear exclusions. Preserve library format/KDF defaults and enforce tested resource limits. Keep key recovery and wallet-data backup as separate statuses and instructions. +For remote device backups, expose retained devices/generations and completion +evidence. Do not select a newer empty device solely by timestamp or begin its +backup writes before recovery source selection and validation finish. Preserve +access to older complete copies and test two successive reinstalls. Inventory +custom app tables even when they share the wallet database; define their separate +recovery contract and disclose missing context instead of claiming full export. For import, recover the expected identity independently, decrypt/validate for preview, compare identity and network, then confirm target and explicit mode. @@ -72,6 +78,9 @@ resource limits and clean-device/provider-loss recovery. Verify transaction and derivation records, relationships, tombstones and binary data; balance alone is insufficient. Exercise restart and a controlled synthetic/testnet spend. Test each claimed runtime and cross-wallet direction with exact versions. +Include a restored synthetic BRC-29 output: sign a noSend spend, verify its +unlocking script locally against the independently retained source output, then +abort the isolated test action and verify cleanup without broadcasting. Deliver implementation, user instructions, recovery inventory, acceptance evidence and explicit unresolved limits. Distinguish implemented, locally diff --git a/docs/guides/wallet-recovery-drill.md b/docs/guides/wallet-recovery-drill.md index fd5507c3c..52ab070b1 100644 --- a/docs/guides/wallet-recovery-drill.md +++ b/docs/guides/wallet-recovery-drill.md @@ -3,8 +3,8 @@ id: wallet-recovery-drill title: 'Wallet Recovery Drill and Acceptance Checklist' kind: guide version: '1.0.0' -last_updated: '2026-09-24' -last_verified: '2026-09-24' +last_updated: '2026-09-29' +last_verified: '2026-09-29' review_cadence_days: 30 status: stable tags: [wallet, recovery, backup, testing, interoperability] @@ -101,6 +101,9 @@ which product data is excluded and how it is restored separately. | Concurrent source writes | Export uses the documented consistency procedure; no claim that sequential reads form a database snapshot | | Low-memory device and large file | Enforced product limits, measured peak memory/time and usable failure/progress UI; no weakened cryptography | | Browser eviction, app uninstall, device loss | Recovery succeeds through independent copies, or the exact unsupported case is clearly disclosed | +| Two reinstalls; newest device empty or incomplete | Older complete records remain recoverable; source selection and validation finish before the new device starts backup writes | +| Records split across device histories | The host's supported replay preserves the expected union and conflict semantics, or clearly reports the unsupported recovery path | +| Custom tables beside wallet tables | Standard wallet records recover; excluded application context is disclosed and its separate restoration is tested where supported | | Vendor A export → vendor B import | Same identity/network and compatible row semantics; unsupported product extensions disclosed; test the reverse direction when claimed | | Key/share or passphrase dependency unavailable | Document the supported combinations and actual failure behavior; no circular recovery assumption | @@ -111,6 +114,24 @@ loss. Keep monitor/background jobs disabled until their side effects and replay state have been reviewed. Replicas, backups and service health are separate evidence. +### Prove restored derivation without broadcasting + +For a synthetic BRC-29 fixture, retain the expected source outpoint, locking +script, sender identity and derivation metadata independently of the archive. +After import, use the recovered wallet to prepare and sign a spend with +`noSend: true`, then verify the unlocking script against that original output +locally. A matching balance, identity or returned txid alone does not prove that +the output's derivation metadata survived. Record the actual no-send capability +and exact package versions; do not assume an older wallet exposes current APIs. + +Keep the restored copy's monitor, network broadcasting and provider writes +disabled through tested host controls. Release the test action with the +supported `abortAction` path and verify the isolated target's inputs are +available again. This rehearsal uses synthetic data; it is not permission to +sign against a user's live funds or proof that a physical wallet reinstall +succeeded. Product-owned context and custom signing modules need their own +acceptance evidence even when this signature check passes. + ## Diagnose failures without destroying evidence | Symptom | Safe next step | @@ -137,11 +158,13 @@ Runtime / device / network / backend / schema version: Synthetic fixture ID / profile count / data classes covered: Key recovery method / unavailable factors tested (no secret values): Data recovery method / consistent recovery point / archive digest: +Backup source device / generation / completion evidence / alternatives retained: Product data included / excluded / separate restoration method: Source device/provider dependencies removed for the drill: Restore or merge target and authority decision: Expected vs actual inventory / relationships / binary-data verification: Identity / read / controlled spend / restart / retry outcomes: +BRC-29 no-send signature verification / action cleanup / broadcast isolation: Negative, resource-limit and cross-wallet scenarios with results: Observed data-loss interval / restore duration vs declared objectives: Unresolved gaps / user-visible limitations / remediation owner: diff --git a/docs/guides/wallet-sync-reliability.md b/docs/guides/wallet-sync-reliability.md new file mode 100644 index 000000000..28c527560 --- /dev/null +++ b/docs/guides/wallet-sync-reliability.md @@ -0,0 +1,1202 @@ +--- +id: wallet-sync-reliability +title: 'Resumable wallet synchronization and proof recovery' +kind: guide +version: '1.0.0' +last_updated: '2026-09-30' +last_verified: '2026-09-30' +review_cadence_days: 30 +status: beta +tags: [wallet, sync, storage, performance] +--- + +# Resumable wallet synchronization and proof recovery + +The unpublished Toolbox 2.15 candidate provides a complete bounded pull-sync +session for local Knex and IndexedDB destinations. Source requests, proof-provider +I/O and progress callbacks run outside exclusive page ownership; the destination +commits each page and its checkpoint atomically. Foreground reads and writes can +proceed between commits. Remote/custom destinations retain the established +exclusive path unless their local provider explicitly supports the required +capability. Existing `syncFromReader`, `syncToWriter` and their result contracts +remain available. + +## Consumer contract + +```ts +const controller = new AbortController() +const result = await manager.syncFromReaderResumable(identityKey, source, { + signal: controller.signal, + maxItems: 1000, + maxRoughSize: 262144, + onProgress: progress => { + // Render counters and state; keep this callback short and non-throwing. + console.log(progress.state, progress.pages, progress.inserts, progress.updates) + } +}) +``` + +The result reports `completed` or `cancelled`, actual `paged` or `exclusive` +execution, acknowledged page/entity counts and the last durable checkpoint. +Progress includes reading, preparing, committing, committed and terminal states, +with separate source-read, preparation, queue and commit timings when available. +A callback exception rejects the call; resume from the destination rather than +assuming that no page committed. + +Cancellation is cooperative. It waits for an in-flight source read or proof +lookup to drain, discards an uncommitted page, and waits for an already-started +commit acknowledgement. It does not abort a database transaction after its +outcome becomes uncertain or undo acknowledged data. Invoke the API again to +resume; do not use UI counters as a checkpoint. A lost acknowledgement rejects +without automatically replaying a write, and the next session loads the durable +destination state. The retained local SQL path distinguishes resuming a still-live +view from restarting a replacement view, as described below. + +Only one page is in flight. The new API defaults to 1,000 rows and 262,144 rough +encoded bytes (256 KiB). Options must be positive safe integers, no greater than +1,000 rows and 10,000,000 rough bytes. Existing sync methods retain their prior +defaults. The new default is a measured starting point for constrained clients, +not a universal memory ceiling. A single large record can exceed a rough page target; +the existing separately negotiated 64 MiB transfer-frame bound still applies. +See [bounded transfers](https://github.com/bsv-blockchain/ts-stack/blob/main/packages/wallet/wallet-toolbox/docs/sync-transfer.md). +There is no unbounded prefetch queue or hidden parallel merge. + +## Identity, network and persistence safety + +Missing, unrecognized or mismatched chain declarations fail with the existing +`WERR_NETWORK_CHAIN` before sync state or user writes. Live sync never infers a +provider's chain. Legacy providers without access capabilities retain exclusive +ownership; they still need matching chain declarations. Both sync directions +honor returned `ProcessSyncChunkResult.error` values as well as thrown errors. +Neither path advances counts or checkpoints after an error, and a nonterminal +page that makes no checkpoint progress fails explicitly. + +A completed unchanged backup may reset all twelve legacy offsets to zero while +retaining its `since` timestamp. Clients accept that reset only with literal +`done: true`; nonterminal and partial offset retreats still fail. State identity +and timestamp monotonicity are checked on terminal replies too. This fixes the +`Invalid sync checkpoint` error on repeated ordinary HTTP backups without +changing checkpoint JSON or persisted state. A lost terminal acknowledgement +still resumes by loading the destination's durable checkpoint. A returned +provider failure takes precedence over any accompanying checkpoint fields. + +Sessions pin wallet identity, destination instance and manager generation. +Switching/destroying the destination fences late replies before mutation. The +page commit rechecks the durable checkpoint and any proof rows inspected during +preparation, preventing concurrent sessions or monitor repairs from overwriting +newer state. Input pages and checkpoint values are detached before asynchronous +validation. Legacy checkpoints retain their schema; the durable local SQL path +below adds auxiliary persistence. No archive envelope changes. + +The queue permits at most eight concurrent readers on providers advertising safe +reads. Writers remain exclusive. Foreground work has priority, with bounded +bypass (eight grants or one second of queue age) so background sync can still +progress. Legacy providers remain exclusive. Advertising a capability is a +provider correctness promise, not merely a performance hint. + +The adaptive controller separates fixed page latency from marginal record cost, +uses a bounded sample window, starts with 64 records, limits growth to 2x and +caps proof pages at 128 records and metadata pages at 1,000. A prolonged one-row +floor probes recovery. These bounds prevent the previous fixed-latency feedback +loop from permanently collapsing page size, without treating a slow proof page +as evidence that all later metadata is equally expensive. + +## Canonical proof recovery + +Assembled BEEF is checked before spending or broadcasting. An invalid root can +trigger bounded recovery through the configured chain tracker and active-chain +header/proof provider. A block hash alone does not establish canonicality. +Recovery validates the path, root, height, transaction identity and actual +80-byte header; it preserves transaction bytes and txid-only ancestry. Valid +BEEF takes the fast path without database proof lookups or graph copying. + +At most eight proof lookups run concurrently, and SQL lookups use batches of at +most 250 distinct transaction IDs. Built-in stores atomically compare previous +proof authority fields before persisting a correction and invalidate prepared +BEEF in the same transaction. Delayed monitor responses cannot overwrite newer +proofs. Custom stores lacking compare-and-set support can repair the outgoing +BEEF without claiming that the shared proof record was persisted. + +Unavailable canonical evidence produces `WERR_INVALID_MERKLE_ROOT` with its +existing public/JSON identity and retry guidance. Recovery does not fabricate +success or broadcast a transaction. Historical stores still need independent +operator auditing; an upstream fix is not evidence that every deployed record +was repaired. + +## Timestamp boundaries and snapshot scope + +The legacy timestamp/offset path preserves BRC-40's inclusive `since` boundary and existing entity order, +ID mapping, tombstones and merge rules. Records arriving later with the same +last timestamp must remain discoverable. Consequently, a large imported batch +sharing one timestamp can require a complete boundary reread even when it has +no changes. The benchmark deliberately tests this case. Advancing beyond that +boundary without a coherent source contract would lose valid peer changes. + +A successful live merge is an eventual replica, not a point-in-time source +snapshot or authorization to activate a new primary. Existing BRC-38/BRC-39 +export/import contracts are unchanged. Issue #544 remains open for coherent +remote snapshots, streamed archives and consumer adoption. + +## Retained local SQL read views (unpublished candidate) + +The 2.15 candidate adds `StorageKnex.openReadSnapshot({ lifetimeMs, signal })`. +Opening pins a SQLite or MySQL read view before returning. Its `read(callback)` +method reuses that transaction across idle periods; pass the supplied token to +**every** query and await every query before returning. Each provider admits one +retained view and each view admits one read at a time. Concurrent/nested reads +reject immediately. These local methods are excluded from the storage RPC +allowlist; the handle is neither an authenticated remote export nor a profile +boundary. + +The lifetime defaults to five minutes, includes acquisition, and accepts integer +milliseconds from 1 through 3,600,000. Both monotonic and wall clocks enforce +expiry. Close, cancellation and expiry stop new reads and discard late results. +Await `close()` or `closed` to observe physical transaction/connection cleanup; +a read failure invalidates the view and rolls back. Configure database query and +connection deadlines separately: cancelling the view cannot interrupt a hung +driver call or user callback. Do not perform writes, peer/file I/O, progress +callbacks, or await `close()` inside a read callback. + +A view occupies one pool connection until cleanup. With a one-connection pool, +other work must wait; the API does not create an independent writer. SQLite +concurrent-write qualification uses WAL and a separate connection. MySQL uses a +read-only repeatable-read transaction without changing pooled session defaults. +SQLite relies on the trusted callback's read-only contract. `StorageIdb` and +unsupported providers explicitly refuse retained views; their existing scoped +snapshot behavior is unchanged. Process exit loses the view. +The lifetime limits retention time, not the bytes accumulated in database WAL or +undo history while other writers continue; database storage limits remain separate. + +The primitive itself does not provide a durable cursor, +remote handle, concurrent IndexedDB snapshot, or streaming archive. The local +sync integration below adds destination persistence and scheduling separately; +the retained-view primitive itself does not change legacy index order or wire bytes. + +### Profile-bound local SQL pages + +`StorageKnex.openWalletReadSnapshot(identityKey, { lifetimeMs, signal })` adds a +version-one typed paging interface over the same retained lifetime. Inspect +`supportsWalletReadSnapshot()` first; the base provider and IndexedDB explicitly +refuse it. The API and its capability are excluded from RPC. These are trusted +local database operations, not remote authentication or complete archive validation. + +The returned source settings, user and all thirteen standard tables share one +view. No primary activation or managed-change policy is applied. Proofs include +those referenced only through the user's transaction requests. Deleted labels, +tags, mappings, baskets and certificates remain present. Inconsistent profile +ownership across mappings or certificate fields rejects the view rather than +silently omitting a relationship. Binary columns are `Uint8Array`, never expanded +number arrays; stored scripts remain stored values, without reconstruction. + +```ts +const view = await storage.openWalletReadSnapshot(identityKey, { signal }) +try { + let cursor + for (;;) { + const page = await view.readPage('transactions', cursor, { + maxRows: 128, + maxBytes: 262144 + }) + await consumePackedRows(page.rows) + if (page.done) break + cursor = page.cursor + } +} finally { + await view.close() +} +``` + +Each cursor binds the view and table. Retrying the same position and limits +repeats a page. Process loss, cancellation, close or expiry invalidates that +position; a new view cannot accept it. It is not a durable destination checkpoint. +One read is allowed at a time and each provider retains only one SQL view. +The existing pool, query-deadline and lifetime limits above still apply. + +Pages default to 128 rows and a 262,144-byte payload charge; limits are integer +values from 1 to 1,000 rows and 1 to 16,777,216 bytes. SQL first returns bounded +keys and cell lengths. The charge is twice each cell's stored byte length plus +64 bytes per cell. Only the prefix fitting both limits is fetched. A first row +over budget explicitly rejects, without loading or skipping that row. This +charge bounds stored page payload, not encoded wire size or measured process +RSS; callers must release consumed pages. Large-value streaming remains required +for records exceeding the maximum budget, and header/schema metadata is separate. + +Traversal preserves the original unique-key order, including label/tag-first +mapping keys and field-name-first certificate keys, with each database's +collation. This order differs from canonical BRC-38 array order. The auxiliary +profile index below adds bounded profile range selection for eight direct tables +without altering any standard-table index, legacy OFFSET order or cursor bytes. +The separate numeric relation migration extends that selection to label and tag +maps. Certificate-field/global-table query work, commit-order incremental high-water +positions, IndexedDB retention and large-value streaming remain required by the +active program. The local sync integration below consumes these pages. + +### Auxiliary profile indexes (unpublished candidate) + +Migration `2026-10-01-003 add snapshot profile key indexes` adds +`snapshot_profile_keys` and `snapshot_profile_index_progress`. The first table +holds `(table, user, row)` keys for transactions, outputs, certificates, labels, +baskets, tags, commissions and sync states. Source-table insert, key/profile +update and delete triggers maintain the keys in the writer's transaction, +including writes from older binaries and independent connections. The standard +tables and indexes remain intact. These auxiliary tables are not wallet archive +content and do not change BRC-38, legacy sync or snapshot cursor encodings. + +Use the normal provider migration entry point. This migration intentionally +uses `transaction: false`: MySQL DDL commits independently, and each bootstrap +batch commits its keys and progress together in a separate transaction on both +backends. A batch reads at most 256 source rows. SQLite takes its writer lock +before reading progress; MySQL locks the current progress and source rows until +commit. Triggers precede bootstrap, so independent profile moves, deletes and +inserts behind a committed cursor remain visible to resumed indexing. No +standard table is rebuilt and no wallet row is rewritten. + +An interruption can leave auxiliary tables, triggers or committed bootstrap +batches before the migration journal entry. Retain them and retry the migration; +matching objects and committed positions are reused. Mismatched definitions or +malformed progress refuse completion. An abruptly terminated migrator may also +leave Knex's migration lock claimed. Before recovering that lock, independently +verify that its migrator has stopped and exclude every other migrator; this +implementation does not automatically break a migration lock. Keep a verified +backup and use the existing migration recovery procedure. + +A retained reader chooses its mode from the exact migration journal entry and +complete progress in the same view as its header and pages. Missing journal +entries retain the previous traversal. A journaled but incomplete or mismatched +auxiliary schema refuses the new view. Existing views keep their original mode +across later migration or writer commits. Ordinary pages, archive pages and +archive closure checks share that choice; no private mode flag is added to +public metadata. + +Drain snapshot readers and exclude concurrent migrators before rollback. The +migration's down operation validates owned objects, stops insert/update producers +before deleting their observers, then removes only the auxiliary tables and its +journal entry. Standard wallet rows and indexes remain unchanged. Older binaries +may read and write the unchanged standard schema while the forward migration's +triggers remain installed; binary downgrade procedures still apply to the +candidate's other migrations. + +### Auxiliary numeric relation indexes (unpublished candidate) + +Migration `2026-10-01-004 add snapshot relation key indexes` adds +`snapshot_relation_keys` and `snapshot_relation_index_progress` for +`tx_labels_map` and `output_tags_map`. Keys preserve label/tag-first composite +cursor order. Each key records the independent left and right ownership bases: +a mapping belongs to either parent's profile, including tombstones and inconsistent +cross-profile mappings. Such inconsistencies remain visible to the existing +page/closure refusal; the index must not filter them out and silently export less +data. Parent moves, key updates and physical deletion remove only the affected +ownership basis. + +All removal observers across both relationship graphs are installed before any +producer. Bootstrap starts after the complete trigger set exists and commits at +most 256 mapping rows with its composite progress position. MySQL locks current +mapping rows and parent owners through commit. Trigger producers use current +locking reads even if their writer transaction has an older consistent read view. +SQLite uses its writer lock. Standard tables and their indexes are unchanged. +MySQL page queries explicitly select the auxiliary primary index: a maintenance +index can otherwise scan and sort an entire profile before applying the page limit. +The query also reads the recorded membership and fixes the join order and indexed +source lookup. A covering-index plan can otherwise scan the prior prefix before +InnoDB refreshes statistics, even when ordinary EXPLAIN reports a range. + +The same migration-journal, retained-view, interrupted-migrator recovery and +reader-drain requirements described above apply. The migration has +`transaction: false`; preserve partial owned objects and committed progress, then +retry only after excluding another migrator. MySQL index DDL can stop between +table and index creation. Compatible missing indexes resume; mismatched table, +index or trigger definitions refuse adoption or removal. Down validates all +objects before removing producers, observers and auxiliary tables. Drain readers +before down; standard rows, legacy OFFSET order and portable/cursor bytes remain +unchanged. These auxiliary tables are excluded from BRC-38. + +Repository fixtures compare all thirteen ordinary/archive tables and legacy +offsets before and after indexing, generate ownership/rekey schedules, and kill +the real migrator at seven DDL/bootstrap boundaries on SQLite and MySQL. The +native MySQL fixture observes independent locks under READ COMMITTED and +REPEATABLE READ and checks late-page native row-read counts with interleaved +profiles both before and after ANALYZE TABLE. +These are isolated synthetic fixtures, not deployed PXC or physical mobile +qualification. The following migrations cover certificate fields and global +proof requests/proofs. Source commit ordering, nonblocking IndexedDB, streaming +and staged restore remain required. Reader advertisement stays disabled and +#569 remains open. + +### Auxiliary certificate-field indexes (unpublished candidate) + +Migration `2026-10-01-005 add snapshot certificate field key indexes` adds +`snapshot_certificate_field_keys` and `snapshot_certificate_index_progress`. +Each field retains direct-user and parent-certificate membership independently, +including orphan fields and inconsistent ownership that the existing closure +checks must reject. Field/user/key changes, parent moves and physical deletion +update only the affected membership. Case-only and collation-equivalent renames +preserve the exact source field text. + +The auxiliary text key preserves the source field's ordering. MySQL copies its +`varchar(100)` character set and collation and requires transactional InnoDB +source and auxiliary tables. SQLite verifies the source's complete ascending +unique key and built-in BINARY, NOCASE or RTRIM collation. A differently collated +secondary index cannot redefine the cursor. Unsupported or inconsistent schema +metadata refuses migration instead of silently changing source ordering. +Standard tables, indexes, legacy OFFSET order and BRC-38/cursor encodings are +unchanged. Auxiliary tables remain excluded from portable archives. + +All removal observers precede producers. Bootstrap commits at most 256 source +fields and its text/composite position together; a separate started bit keeps an +empty field name valid. MySQL locks current source fields and parent ownership +through commit, including when an independent writer has an older consistent +read view. Parent maintenance uses the auxiliary certificate prefix and exact +source key rather than scanning all source fields. + +The migration has `transaction: false` and resumes partial owned DDL and committed +bootstrap positions. The migration journal and complete progress must both exist +inside the retained read view before ordinary pages, archive pages or closure +checks adopt the index. Missing journal entries retain the previous path; +journaled incomplete state refuses opening. Exclude other migrators before +recovering a verified stopped migrator's lock. Drain retained readers before +down; validate owned objects, remove producers before observers, and preserve all +standard rows and indexes. Keep verified backups and the candidate's other +binary-downgrade restrictions. + +The repository includes generated text/ownership schedules, independent MySQL +writer-lock checks, malformed metadata refusals and actual migration-process +termination at seven DDL/bootstrap boundaries on both databases. Native reader +fixtures measure first and late pages before and after optimizer statistics refresh. These +are synthetic source qualification fixtures, not deployed or physical mobile +acceptance. The following migration covers proof/request indexes; commit +ordering, nonblocking IndexedDB and the remaining sync/portability program +remain open. + +### Auxiliary global proof/request indexes (unpublished candidate) + +Migration `2026-10-01-006 add snapshot global reference indexes` adds four +auxiliary tables: `snapshot_global_edges`, `snapshot_global_keys`, +`snapshot_global_guards` and `snapshot_global_index_progress`. Requests belong +to profiles with a transaction whose txid matches the request. Proofs belong +through either a transaction's direct proof reference or a matching request's +proof reference. Multiple transactions and both reference bases retain separate +edges; removing one basis cannot remove another profile's remaining reference. +Missing proofs keep bounded presence metadata so later insertion or deletion +updates the same indexed selection. Unused proof guards are collected after the +last reference disappears. + +Triggers serialize reference counts and proof presence in the writer's atomic +transaction. MySQL uses current locking reads, including an auxiliary lock for an +absent proof, so an older REPEATABLE READ snapshot cannot publish stale presence +or restore an obsolete request pointer. Removal observers precede producers. +Bootstrap locks at most 256 current transaction rows and commits their edges and +progress together. Retrying a committed batch does not increment reference counts +twice. Source txid comparison metadata must agree; source indexes, exact text, +standard rows, legacy OFFSET and portable/cursor encodings remain unchanged. + +MySQL requires the standard unsigned key and matching text definitions on +transactional InnoDB tables. Custom CASCADE or SET NULL foreign-key actions are +refused before auxiliary DDL because InnoDB does not invoke affected child row +triggers for implicit cascades. Standard RESTRICT and NO ACTION definitions are +supported. SQLite verifies standard key/column definitions and complete txid +lookup indexes with matching BINARY, NOCASE or RTRIM order. Invalid legacy keys +or oversized text refuse bootstrap without publishing completion. + +This is another explicit `transaction: false` migration. Preserve partial owned +DDL and committed positions, exclude other migrators, and recover a stale Knex +lock only after proving its migrator stopped. Ordinary and archive readers adopt +complete journal/progress state inside their pinned view; absent journals keep +legacy selection and incomplete journaled state refuses opening. MySQL pages +use the auxiliary `(table, user, present, row)` index and indexed source lookups, +including before statistics refresh. Drain readers before down; validate all +owned objects before stopping producers and removing auxiliary tables. These +auxiliary structures remain outside BRC-38. + +Fixtures cover shared references, full-width unsigned IDs, source collations, +independent writer commits, retained ordinary/archive pages, bootstrap replay, +eight migrator process-loss boundaries, and first/late query plans and measured +row fetches. The pinned native launcher runs every archive/profile/relation/ +certificate/global family sequentially with a 60-second child deadline per +group and verified owned-container cleanup. The global family is split into +complete process-loss, locking, schedule, seek and integration groups after its +combined local run consumed 51.6 seconds of the 60-second allowance. Every earlier +case remains mandatory. Complete exact-head qualification remains mandatory; +these synthetic fixtures do not establish deployed PXC or physical mobile +acceptance. Commit ordering, tombstones, primary reconciliation, nonblocking +IndexedDB, remote destinations, streaming/staged portability and full system +acceptance remain open. Reader advertisement stays disabled. + +### SQLite conflict-safe index generation (unpublished candidate) + +Migration `2026-10-02-001 repair snapshot SQLite conflict maintenance` replaces +SQLite auxiliary index maintenance with metadata-bound conflict witnesses. +SQLite REPLACE can remove a displaced row without invoking its deletion trigger; +the new BEFORE observers retain displaced IDs and owners before the source write. +AFTER observers reconcile the actual source state and preserve independent +relation, certificate and proof-reference memberships. Supported unique keys use +their existing BINARY, NOCASE or RTRIM comparison; NULL values retain SQLite's +unique-key behavior. No-op and unrelated payload updates avoid auxiliary writes. +Standard rows, indexes, legacy OFFSET order and portable/cursor bytes stay intact. +MySQL continues to use its existing maintenance. + +Installation validates the complete prior migrations, source definitions and +owned auxiliary objects, then atomically installs a fresh v2 generation and +invalidates the previous progress states. Ownership binds both object type and +name: a view sharing an owned trigger name remains a foreign reader. Foreign +views or triggers that depend on legacy auxiliary data refuse installation +before those tables can be retired. Orphan generation objects and unsupported +composite comparison or source-order definitions also refuse adoption. +Twelve source streams copy at most +256 rows per transaction with typed durable positions. Independent writes, +including inserts below the saved cursor, remain observed throughout the copy. +Each completed page yields before the next transaction. Once the copy completes, +obsolete owned tables retire in batches of at most 256 physical rows; only empty +tables are dropped. Source tables are never retired. The hidden SQLite row +identity bounds retirement even when several index families share one logical ID. + +Run `migrate()` explicitly, exclude concurrent migrators, and preserve owned +schema/progress after interruption. Recover a stale Knex lock only after proving +the migrator stopped. The real migrator publishes its journal after copy and +retirement; partial state uses source-query fallback in new readers. Complete +journal/progress state selects v2 indexes inside the same retained view as all +pages. Publication lookup uses the explicitly configured SQLite migration-journal +schema; an identically named journal in another attached database cannot publish +this generation. An already-pinned WAL reader retains its original view across retirement. +Older binaries cannot adopt invalidated progress: do not downgrade their snapshot +implementation against this schema. Ordinary migration rollback refuses without +deleting source rows; use a separately designed forward migration. The explicit +`dropAllData()` API still deletes all wallet data, including an unpublished +partial generation. It is not a recovery or downgrade operation. + +Qualification includes generated conflict schedules, actual wallet schemas, +independent WAL writers, retained ordinary/archive readers and real registered +migration process loss through installation, copying, retirement and publication. +The populated diagnostic uses 13,000 small synthetic source rows and verifies +bounded retirement and six indexed late-page shapes. It does not establish +large-wallet latency, internal SQLite row visits or an overall storage quota. +Freed SQLite pages may remain allocated for reuse; physical file shrink is not +promised. Complete exact-head native, mutation, platform and hosted gates remain +required. Reader advertisement and the full #544 acceptance status are unchanged. + +## Durable local SQL sync and ordinary backup + +With the version-one migration applied, supported local SQL providers use these +pages for ordinary `syncFromReader`, `syncToWriter` and `updateBackups` calls. +`syncFromReaderResumable` and the additive `syncToWriterResumable(auth, writer, +options)` expose cancellation and page progress. Source reading and proof +preparation run outside manager ownership; only destination admission and each +atomic page commit enter the fair background queue. Foreground reads and writes +can proceed between those commits. Existing `progLog` callbacks receive each +committed page and the completion summary, including during serialized fallback; +their returned text remains part of the ordinary result log. Backup destinations, +entity dependencies, proof checks and 128-ID SQL batches run sequentially through +the lazy series coordinator. A failure stops before the next operation starts; +neither parallel database work nor an eager promise queue is introduced. +Primary reconciliation still uses its existing +exclusive path and remains required work in the full program. + +The source gets a dedicated one-connection pool, preserving the original pool's +foreground capacity and telemetry configuration. SQLite requires a file-backed +WAL database. MySQL requires a static connection configuration with a database; +dynamic connection factories, externally shared pools and unsupported providers +retain the existing serialized path. Each provider admits one opening/live sync +source until physical cleanup completes. The default five-minute retention limit +includes opening; `snapshotLifetimeMs` accepts 1–3,600,000 milliseconds. Driver +query deadlines and WAL/undo disk limits remain separate. + +Migration `2026-09-30-001 add durable snapshot sync` adds three auxiliary tables: +`snapshot_sync_sessions`, `snapshot_sync_ids` and +`snapshot_sync_primary_epochs`. A database trigger increments the primary epoch +on every primary change, including an independent writer changing away and back. +It does not change standard-table key order or legacy `sync_states.syncMap` JSON. +SQLite commits the migration atomically. MySQL's idempotent table, foreign-key +and trigger installation can resume after partially committed DDL. Apply +migrations using the normal provider migration entry point before enabling the +capability; merely opening storage does not upgrade its schema. MySQL migration +credentials need permission to create the auxiliary tables, foreign keys and +trigger; the ordinary runtime connection does not install them implicitly. + +A session binds the wallet identity, source/destination storage identities, +network, source view, selected primary and its epoch. Rows, normalized ID mappings +and the destination cursor commit together. Source user metadata joins the first +page commit; cancellation before that commit leaves the selected primary intact. +Push and backup merge the source view's stored primary metadata using the existing +timestamp rule; they do not replace it with a manager's older cached selection. +Pull retains the destination manager's selection. Serialized fallback preserves +the same directional rules. This compatibility behavior does not refresh the +manager cache or implement primary reconciliation. +A queued `setActive` request checks the selected provider after earlier switches +finish. For example, a request to reselect A while a switch from A to B is pending +waits for that switch, then selects A. The same ordering holds when the earlier +switch fails. A progress formatter that throws before a transition starts leaves +existing sync sessions valid. If a transition fails after changing a store, the +manager discards cached primary authorization and reloads persisted selections +before subsequent access. Conflicting selections refuse active authorization +until a later successful selection reconciles the stores. A failed recovery +reload releases ownership so later requests can retry. Primary reconciliation +still uses the serialized copy loop; bounded reconciliation remains part of the +implementation program. +A prepared page is single-use and stale checkpoints reject. If an acknowledgement +is lost while the same source view remains alive, read the destination checkpoint +and resume it. When the source view is lost, open a new view and restart traversal +from the first table; committed mappings and entity merge semantics prevent +repeated inserts. Manager calls own and close their source view, so calling a +manager sync method again starts a replacement view. Advanced callers retaining +the same local capability handle can resume its acknowledged cursor. No source +cursor survives source process loss. Returned acknowledgements must match the +session bindings, next sequence, table position, completion flag and page cursor; +a mismatch rejects before another read or progress count is accepted. + +One packed page is in flight. The resumable APIs default to at most 1,000 rows +and 262,144 charged bytes; configurable ceilings are 1,000 rows and 10,000,000 +bytes. Ordinary push, pull and backup retain their 1,000-row/10,000,000-byte +ceilings; the adaptive controller may request smaller pages. Per-page mapping +work is limited to 4,096 distinct IDs, queried and persisted in batches of 128. +An oversized row, reference limit or retention expiry closes the source and uses +the established serialized fallback, preserving progress counts. Malformed rows, +profile/network mismatches, stale sessions and I/O failures reject. Fallback can +hold manager ownership for the remaining copy; bounded large-value streaming is +still required. The result/progress `mode` reports the selected behavior. + +`snapshotCheckpoint` is separate from the legacy `checkpoint` field. It records +this destination's durable position, not transferable authorization. The local +`getSnapshotSync()` capability is excluded from RPC, and its types do not imply +an IndexedDB, remote or native adapter. The twelve replica tables preserve the +existing merge policy; original source sync-state history remains an archive +concern. BRC-38/39 bytes are unchanged. + +For forward rollback, construct `StorageKnex` with `snapshotSync: false` on both +sides and retain the additive schema. Existing sync APIs then use their established +paths. Stop all new sessions before any binary downgrade; older migration code +may refuse a database containing newer journal entries, so disabling this feature +on a current binary is the supported operational rollback. Do not drop auxiliary +mapping/session state while work is active. No package is published or provider +upgraded by this source change. + +## Next-stage remote design checkpoint (not implemented) + +A future source-snapshot capability should negotiate a versioned contract +separately from ordinary sync and bounded transfer. A snapshot must bind the +authenticated wallet, network, storage identity, source schema, entity ordering +and immutable high-water position. Use a stable per-entity cursor (including a +tie-breaker) within that immutable view; do not reinterpret live timestamp/offset +checkpoints as coherent snapshots. The exact wire names remain subject to shared +conformance review and independent provider implementation. + +Creation must establish the entire snapshot consistently, using a transactional +read view or immutable staging with an atomic manifest. Bound retained snapshots, +bytes, lifetime, per-identity concurrency and cleanup across replicas. A durable +resume token must bind the snapshot digest and cursor, reject another identity, +and fail explicitly after expiry. A restarted reader may resume the same +immutable snapshot or restart a new one; it must never combine both silently. + +Streaming BRC-38 encoding should preserve canonical record order, binaries, +nullable history, source provenance and the standard's deliberate exclusions. +BRC-39 streaming must preserve the released envelope, KDF and authentication: +stage decrypted bytes privately, verify the complete authentication tag and +semantic relationships, then atomically activate an isolated import. Partial +plaintext is never active wallet data. Bound memory, worker/IPC queues, KDF work +and temporary disk independently; record crash recovery and safe cleanup. + +Release gates for that next stage are old/new independent reader/writer vectors, +wrong password/identity/network and corruption cases, same-timestamp/tombstone +relationships, restart at every durable boundary, repeated import, native +browser/mobile evidence and wallet adoption. No partial archive or optimistic +activation should be labeled complete while those gates remain outstanding. + +## Shared SQL snapshot staging (unpublished internal component) + +The `2026-09-30-002 add snapshot archive staging` migration adds three auxiliary +SQL tables for the remote snapshot implementation in progress. It does not change +standard wallet tables, their indexes, legacy checkpoints or BRC-38/39 bytes. +Storage opening still does not run migrations implicitly. The tables do not enter +portable archives or ordinary synchronization. + +`storage/snapshot/archive/KnexSnapshotArchiveStore` is an internal persistence +component, not an authenticated remote export API. A capture owns an internal +writer token and records one original source view, schema, network and wallet +profile. Each bounded page, sequence receipt and quota charge commits atomically. +An exact retry repeats the receipt; changed bytes or metadata reject. A completed +capture becomes immutable and readable from another server connection. The +capture controller must validate complete source closure before sealing it; this +store alone does not validate wallet records or cryptographic proofs. + +The internal `captureKnexSnapshotArchive` controller now takes a dedicated SQL +reader and a separate staging connection. It reads original settings, the +profile and the migration version in one retained view; verifies every selected +foreign-key relationship against that profile; and captures all thirteen raw +standard tables through the shared keyset reader. Relationship checks return +only an invalid-row marker, without loading payloads or building full ID maps. +Pages use the existing binary transport frame. They are not canonical BRC-38 +output, and this step does not authenticate transaction/proof contents. + +The controller reads at most 128 rows and a 128 KiB conservative SQL payload +charge per page, then checks the encoded one-MiB limit before staging. It reports +acknowledged page/row/byte counts, releases the source view before sealing, and +discards staging after cancellation or a capture failure. If cleanup itself +fails, the reservation remains available for expiry/reaping recovery. Callers +must supply independent reader/staging pools and keep schema migrations outside +active capture windows. No RPC route or advertised capability is added. + +The store's internal `directory` read returns bounded receipt metadata without +selecting payloads or returning writer credentials. It preserves the exact +original binding JSON bytes used to seed the hash chain. A replacement server +connection can read the same sealed directory; profile authorization and expiry +are checked again after loading its receipts. + +`verifySnapshotArchiveDirectory` validates the expected profile, network, +original storage identity and optional resumed root/schema, exact metadata +fields, lifetime, row totals and the complete ordered receipt chain. All thirteen +table boundaries must be present. It returns detached receipts and a fixed-size +table index, so callers can verify arbitrary-table pages without downloading +earlier payloads. `verifySnapshotArchivePage` copies bounded typed bytes and +checks their digest and metadata against an already verified receipt. Returned +dates remain JavaScript Date objects: freezing their containing objects does not +freeze their internal time values. An eventual adapter must keep its verified +binding private and give callers fresh metadata copies. + +The maximum accepted directory fits below a one-MiB metadata budget. Transport +integration must separately limit incoming envelope bytes before parsing. These +checks establish inclusion under the received root, not independent trust in the +source's data or proof semantics. The authenticated transport below consumes this directory; the remote row reader +and manager integration remain incomplete. + +The internal creation-request store adds the separate +`2026-09-30-003 add snapshot archive requests` migration. A request ID hashes its +version, nonce, immutable absolute deadline and byte reservation. Admission checks +the database clock; the same expired request cannot reopen after its receipt has +been collected. The authenticated transport obtains fresh server time before a caller selects +that deadline. + +Claiming a request reserves the existing shared handle/byte capacity before a +source pool opens. Archive assignment and the handoff from that pending charge +commit together; sealing and ready-receipt publication also share a transaction. +Lost acknowledgements recover the same ready archive. A process that loses an +incomplete source view cannot replace it under the same request. Terminal status +is separate from cleanup ownership: capacity stays occupied until pending charges +or staged pages are released exactly once. Retained receipts are limited to four +per profile and 64 globally, including terminal history; only expired, released +receipts can be collected. Close outstanding requests before removing this schema. + +The request lifecycle is tested through actual SQLite transactions, independent +connections, interrupted assignment/publication and generated retry/cancel +schedules. Separate synchronous SQLite pools in one event loop can return +`SQLITE_BUSY`; tests drain both operations, check that exact refusal and retry +without double release. This does not qualify foreground latency or transparent +contention recovery. + +The internal capture service now uses the provider's existing single owned reader +slot, shared with local sync. It tracks admission synchronously before any await, +claims SQL capacity before pool acquisition, and binds same-request retries to the +same completion or durable receipt. Capture and pool cleanup precede ready +publication. Explicit cancellation and service shutdown wait for owned cleanup; +shutdown fences new captures while completed archives remain available to a +replacement service. Failed physical cleanup fences the controller and retains +its reservation, including failure during source opening before a view is returned. +The source view lasts at most five minutes or the request's remaining lifetime; +the ready archive retains its original fixed deadline. This bounds process-local +reader ownership, not distributed physical pools through arbitrary process loss +or unbounded driver cleanup. The HTTP layer below exposes this controller; a +remote row reader and manager adoption remain separate work. + +The initial policy allows at most eight handles and 128 MiB of logical reserved +storage globally, one handle and 32 MiB per profile, 1 MiB per page, 1,000 rows per +page and 4,096 pages. Metadata is at most 64 KiB. A reservation includes encoded +metadata/payload bytes plus fixed header and page charges; it is not a measured +bound on SQL file size, transaction logs, temporary disk or process RSS. The +shared database clock controls expiry (five minutes by default, at most one +hour). These preliminary limits do not establish acceptance for larger wallets; +operator resource policy and measured storage costs remain required work. + +Partial, closing, expired or differently owned captures are unavailable to +readers. Cleanup first fences new writes, then deletes at most 32 exact page +keys per statement. The profile and global reservation remain occupied until all +pages are removed. Interrupted cleanup is resumable; concurrent closers release +capacity once. The current component requires its owning controller to invoke +cleanup/reaping. No unattended worker or public capability is installed by the +migration. + +SQLite tests cover 300 generated staging schedules and 300 actual SQL capture/ +cancellation schedules, independent hash-chain +receipts, cross-profile reads and cleanup, exact limits, and failures between page +insertion and checkpoint update. A synthetic process fixture terminates the +writer at five durable boundaries: before data, after page insertion, after the +checkpoint write, after transaction commit and after sealing but before the +acknowledgement. Uncommitted pages roll back; committed pages retain exact retry +receipts; only sealed captures are readable. Isolated MySQL 8.4.11 also exercises +1 MiB pages, independent connections racing for one profile, concurrent cleanup, +rollback and partial DDL recovery. Its controller fixture captures thirteen +tables over fourteen pages, retaining 140 original labels and primary metadata +while an independent writer updates both. It also verifies schema provenance, +packed binary and cross-profile relationship refusal. These results apply to +those fixtures. Deployed PXC, remote row/manager integration, canonical streaming, +complete portable semantic validation and the full #544 program remain open. + +Run the synthetic process-termination fixture from the repository root on macOS +or Linux (Node 24 and the package build are required): + +```sh +pnpm --filter @bsv/wallet-toolbox test:snapshot-archive-crash +``` + +The fixture owns a fresh temporary SQLite database per phase, kills only its own +child writer, checks recovery through a new connection, and removes those files. +It prints one result record per phase. This is a storage persistence check; its +synthetic pages are not a BRC-38 archive or a funded wallet recovery. + +The MySQL fixture uses the local Docker Desktop `desktop-linux` context and the +already available MySQL image +`mysql@sha256:0744ee5ef89ce6ccfa13de3e579fe6b9e27f93dd70da9c06d2c908b1b193fb8d`. +It creates a loopback-only disposable container with a 1 GiB memory limit, two +CPUs, 256 PIDs and a 512 MiB temporary data filesystem, then removes it. The +launcher neither pulls an image nor accepts an external database connection. + +```sh +pnpm --filter @bsv/wallet-toolbox test:snapshot-archive-mysql +``` + +## Reproducible validation + +The fixture uses 10,000 same-timestamp labels (including tombstones), 64 proofs +with 32 KiB transactions and associated wallet transactions. It compares +exclusive capability fallback with paged mode on native Chromium IndexedDB, +SQLite and authenticated HTTP-to-SQLite. The same source state is restored +between modes. It reports full-copy time, CPU/RSS or browser heap, SQL queries, +response body bytes, foreground percentiles, event-loop delays and page timings. +It also checks unchanged-boundary replay, incremental tombstones and a settled +unchanged copy. A 15 ms source delay tests fixed-overhead behavior. + +```sh +pnpm --filter @bsv/wallet-toolbox bench:storage-sync +pnpm --filter @bsv/wallet-toolbox-client bench:sync +``` + +Native Chrome/Chromium is required (`CHROME_BIN` can select it). Emulator tests +remain useful for deterministic storage faults; fake IndexedDB timings do not +represent browser database performance. The acceptance fixture bounds full-copy +time, memory growth and foreground/event-loop p95, and requires foreground p95 +to improve at least 2x without doubling full-copy time. Those are fixture gates, +not latency guarantees for every wallet, device or provider. + +### September 23 local measurements + +Measured sequentially on macOS arm64, Node 24.19.0 and pnpm 10.33.2, using the +256 KiB fixture above. The control is this branch's exclusive capability fallback; +it isolates scheduling behavior, not every change since the released package. +Foreground latency starts when its timer callback runs. Event-loop delay is +reported separately so synchronous CPU stalls remain visible. The exclusive +control has very few completed foreground samples because it holds the queue. + +| Backend / mode | Full copy ms | Foreground samples | Foreground p50 / p95 / p99 ms | Event-loop p95 / p99 ms | Pages | +| ------------------------------ | -----------: | -----------------: | ----------------------------- | ----------------------- | ----: | +| Chromium IndexedDB / exclusive | 4027.1 | 1 | 4021.50 / 4021.50 / 4021.50 | 1.30 / 1.60 | 45 | +| Chromium IndexedDB / paged | 4110.0 | 384 | 0.90 / 25.90 / 132.60 | 1.30 / 1.70 | 45 | +| sqlite / exclusive | 2368.7 | 1 | 2342.43 / 2342.43 / 2342.43 | 85.12 / 106.86 | 45 | +| sqlite / paged | 2373.8 | 133 | 0.17 / 0.32 / 0.40 | 84.86 / 109.29 | 45 | +| http / exclusive | 26184.7 | 5 | 0.25 / 26092.86 / 26092.86 | 250.70 / 271.70 | 47 | +| http / paged | 26394.8 | 319 | 0.24 / 0.35 / 0.40 | 255.93 / 271.98 | 47 | + +Native browser peak JS heap was 79.8 MB exclusive and 56.3 MB paged. SQLite +full-copy CPU was 1.86/1.86 seconds and sampled RSS growth 127.6/60.1 MB. +Authenticated HTTP CPU was 25.93/26.08 seconds, RSS growth 152.1/95.1 MB, +and response bodies 7,173,777/7,172,813 bytes. Values are exclusive/paged; +HTTP bytes exclude headers, requests and TLS framing. HTTP and SQLite include +source and destination in the same process. Absolute RSS includes the test +runner and previously allocated heap; sampled growth is not a total memory cap. + +SQLite issued 31,612/31,876 SQL queries and HTTP issued 32,241/32,866, including +foreground operations. Extra queries reflect completed foreground work; this +change does not claim a database-query reduction. Paged maximum commit times +were 142 ms (browser), 104 ms (SQLite) and 81 ms (HTTP). Maximum measured paged +queue wait was 1 ms in the browser and 0 ms in the local SQL/HTTP fixtures. + +All modes preserved the full same-timestamp boundary reread: 45 pages locally +and 47 over HTTP; a subsequent settled unchanged copy used two pages. The +network fixture remains CPU-heavy, with about 256 ms event-loop p95 despite +short queue waits. Moving crypto/serialization off the main thread is future +work; queue responsiveness must not be represented as eliminating that cost. +These are reproducible local observations, not cross-device performance promises. + +Adjacent local spending benchmarks also pass: the pathological 178-source BEEF +plan used 17 queries and about 198 ms; the eight-source cold/prepared paths used +14 queries and about 7.3/5.8 ms. The funding, action-batch and legacy storage-sync +benchmark gates pass. Their existing external PXC cohorts require their governed +MySQL environment and were not executed by the default local invocation. + +### September 24 review-update measurements + +After integrating the SDK compatibility repair, repeat measurements with official +Node 24.18.0 and the same sequential fixture passed the acceptance gates: + +| Backend | Full-copy ms, exclusive / paged | Foreground p95 ms, exclusive / paged | Paged event-loop p95 ms | +| ---------------------------- | ------------------------------: | -----------------------------------: | ----------------------: | +| Native Chromium IndexedDB | 3943.6 / 3971.1 | 3938.80 / 24.50 | 1.00 | +| SQLite | 2270.1 / 2238.5 | 2247.25 / 0.17 | 82.23 | +| Authenticated HTTP to SQLite | 20982.5 / 21004.3 | 20901.51 / 0.31 | 195.69 | + +These runs include shared reader/writer admission with unchanged authorization +ordering and direct entity normalization. Native peak heap was 79.8 MB exclusive +and 56.8 MB paged. Inclusive boundary replay and the two-page settled unchanged +copy remain intact. The same timing/sample limitations above apply; HTTP crypto +and serialization still contribute event-loop delay. Subsequent forwarding and +immutable-descriptor allocation refinements preserve the measured queue algorithm. + +## Authenticated snapshot archive transport (unpublished candidate) + +`StorageServer` now advertises a version-one `snapshotArchive` capability in its +runtime settings when the provider supports a dedicated static SQLite-WAL/MySQL +reader, the archive/request tables exist, and the configured envelopes can fit +the protocol. `snapshotSync: false` on the provider or `snapshotArchives: false` +on the server disables it. Persisted settings and ordinary RPC behavior remain +unchanged. Migration alone installs neither a listener nor a housekeeping worker. + +Both full and mobile clients expose `getSnapshotArchiveTransport(identityKey)` +after authenticated negotiation; old, disabled or unsupported peers return no +transport. A client can also set `snapshotArchives: false`. This is a low-level +archive transport. Ordinary remote sync and portable export do not yet adopt it. +It does not supply a `WalletReadSnapshot`, a remote destination, canonical BRC-38 +or independently validated transaction/proof semantics. + +The transport obtains a fresh authenticated database-clock offer carrying the +source storage identity, chain and schema. `start` acknowledges durable admission +promptly; `status` polls the same exact deadline-bound request. A disconnected +caller retries that tuple, without extending its deadline or opening a second +source. A replacement server recovers completed receipts/directories/pages from +shared storage. An interrupted incomplete capture cannot resume as a new view +under the old request. `cancel` closes that request; a new capture needs a new ID. + +Every method accepts exactly one versioned, profile-bound argument and rejects +extra ownership fields. The server binds the profile to BRC-103 authentication; +internal claim/writer tokens have no wire representation. The client retains its +shared authenticated server pin and separate storage-identity binding. Tagged +binary responses are mandatory. The dedicated AuthFetch caps response bytes at +2 MiB before framing/parsing, including the JSON-RPC envelope; maximum directory +and one-MiB-page fixtures fit that ceiling after HTML escaping and base64 encoding. +The server applies an additional 4-KiB request-envelope bound after its existing +bounded JSON parser. This is not a claim of a 4-KiB pre-parse allocation limit. + +Creation reaps expired pending, partial, ready and legacy archives before claiming +new capacity. Explicit `resource-limited` terminal receipts distinguish a failed +capture budget from other failures, preserving the first terminal state through +cleanup. Future sync integration may consider serialized fallback before it has +exposed a source or begun a destination session; corruption, authentication, +network failures and mid-stream errors must still fail the active operation. + +Server shutdown fences new captures synchronously and awaits both physical reader +cleanup and the HTTP close callback. A failed cleanup stays observable and keeps +its reservation. Temporary HTTP disconnection does not implicitly cancel an +accepted capture. Tests use real authenticated loopback HTTP with synthetic +SQLite profiles, both client variants, lost admission acknowledgements, server +replacement, profile isolation, rollback combinations, response limits and +shutdown during opening. These fixtures do not establish deployed performance, +physical mobile acceptance or completion of #544. + +## Remote row reader (unadvertised implementation) + +The client now implements a source-only `getSnapshotSync()` adapter that +negotiates the separate `snapshotArchiveReaderVersion: 1` setting when opened. +The server does not yet advertise that setting. Controlled HTTP fixtures opt in +to exercise the implementation; ordinary remote sync continues its existing path. +Old archive capability objects and legacy request/response shapes remain unchanged. +Legacy cancellation can report pending cleanup as an error; the explicit pending +receipt belongs only to the v2 reader. +The new adapter refuses destination operations and does not imply portable +export, staged restore or a remote destination implementation. + +Reader offers retain bounded metadata without acquiring a source. An explicit +capacity refusal returns no request. An offered request uses version two and a +separate digest domain; admission requires that exact persisted offer and never +creates an absent request. Consequently cancellation can collect a successfully +released reader receipt immediately without allowing a delayed admission or the +legacy start method to recreate it. Failed and resource-limited receipts remain +observable until explicit cancellation or expiry. The existing four/profile, +64-total metadata limits and logical archive reservations are unchanged. + +Only a native-fetch rejection permits one retry of an immutable admission, +status, directory, page or cancellation request. An offer is never retried; +recovery neither renews the lease nor starts another capture. Authentication, +framing and RPC failures do not select that retry. +Cancellation recognizes the local signal and the SDK's explicit cancellation +code; concurrent independent errors remain failures. The client uses one fixed +wall/monotonic lease, one operation at a time and one private decoded frame. It +checks the complete directory, receipt and row representation before returning +packed rows, fresh dates and an additive version-one `cursor.archivePosition`. +The position identifies the archive, frame sequence and consumed row offset; it +also checks the original row keys. Table lookup does not fetch earlier tables +or reinterpret SQL collation as JavaScript ordering. + +Local destinations persist that optional position with the atomic row commit. +Preparation and progress callbacks receive detached cursor metadata, and an +acknowledgement with a missing or changed position is rejected. Legacy cursor +JSON is unchanged when the position is absent. The manager forwards cancellation +to source opening, returns committed progress only after successful cleanup and +does not switch to a different copy after an accepted remote source fails. + +Actual loopback HTTP fixtures cover full/mobile negotiation, every archived +table, original-source retention through replacement, native-fetch loss before +and after immutable requests, failed authentication and cancellation. Both +clients also sync over HTTP into an occupied local destination: cancellation +retains the committed archive position, a later view completes without duplicate +labels, and the active destination and unrelated profile remain unchanged. The generated +reader property varies frame and caller-page boundaries over at least 300 cases. +These are implementation checks, not completed performance or platform evidence. +In particular, a cancellation handled by another replica can fence durable +admission and staging without proving that the original replica has physically +drained its SQL reader. Owner recovery and bounded driver/query cleanup remain +open requirements before server advertisement and program completion. No claim +of a distributed physical-pool ceiling follows from logical quota accounting. + +## Durable source cleanup fence (unadvertised implementation) + +The additive `2026-10-01-001 add snapshot archive source owners` migration +registers each service capture's exact internal claim in one of eight shared +slots before source-pool acquisition. The request and its archive remain charged +until the owning controller acknowledges awaited source and pool cleanup. Ready +publication requires that acknowledgement too. Another controller may cancel or +expire the request, but cannot free its pages, logical reservation or slot while +the owner record remains. Each append checks the durable request fence in the +same transaction as its page write. Direct archive close and expiry cleanup obey +the same owner fence; a pending source does not prevent cleanup of other archives. + +A pending low-level close raises `SnapshotArchiveCleanupPendingError`; it does +not return successful cancellation. The v2 reader RPC carries an exact +`{ version: 1, outcome: 'cleanup-pending', requestId }` receipt. Only the existing +`true` acknowledgement proves completion. The higher reader validates that binding +and polls with one separate 30-second wall/monotonic cleanup allowance, at most 32 +attempts, and delays of 100, 200, 400, 800 and then at most 1,000 ms. Each immutable +operation retains its existing single native-fetch-loss retry. Neither source +lifetime nor request identity is renewed. Exhaustion rejects with cleanup still +pending. The deadline aborts outstanding I/O and waits for its settlement; it does +not abandon that I/O through a racing promise. Transports must separately bound +operations that ignore cancellation. Independent errors retain their identity, +and a late valid completion acknowledgement still proves cleanup. A failed local cleanup fences that controller's +admission. Repeating an acknowledgement is idempotent and an old or incorrect +claim cannot release a successor. The owner migration refuses removal while +owners remain. Run migrations before admitting captures, keep all serving +binaries on the same candidate, and stop and drain captures before any downgrade. +Older draft binaries do not enforce this new table; mixed-version capture is +unsupported. Standard wallet tables and BRC-38/39 bytes are unchanged. + +An elapsed lease is a fence, not proof that an old SQL operation stopped. The +backend guard implementation below permits recovery only after a separate proof +of source closure. The eight logical slots do not establish +a global physical-connection ceiling; per-provider physical admission remains +occupied until its acquisition and cleanup settle. Actual deployment replica and +driver limits require separate qualification. The generated two-connection +lifecycle suite exercises cancellation, repeated status/reaping, incorrect and +exact acknowledgements, and preserved reservations across at least 300 schedules. + +Real full/mobile HTTP tests route cancellation through an independently connected +controller while the original source pool is held in physical destruction. They +observe pending receipts, retained quota and successful foreground writes, then +confirm cleanup before the opening cancellation settles. Generated pending and +lost-acknowledgement schedules preserve one signal, bounded attempts, independent +failure identity and released timers/listeners. Same-server MySQL evidence covers +the source-side fence and delayed pool destruction. These tests do not establish +PXC or a global physical-pool bound. + +The HTTP test fixture independently drains each native archive RPC and listening +socket before destroying the primary database pool. It preserves the tested +public close outcome, including rejection with `undefined`, and continues +cleanup after a synchronous close failure. Four native opening-capture cases +verify that both listeners withdraw, capture destruction settles before the +primary connection closes, and the provider releases its source admission. + +## Backend-bound owner recovery (unadvertised implementation) + +Apply `2026-10-01-002 add snapshot archive source guards` through the normal +SQL migration entry point. It adds a guard version to owners and eight persistent +slot bindings. Existing owners default to version zero and are never inferred to +have a guard; they retain their reservation until explicit source cleanup. +New service captures bind their exact claim before opening a view and recheck +ownership, deadline and binding under the backend guard before reading wallet +data. Slot bindings survive release and reuse, preventing a delayed claimant +from choosing a new independent guard. + +For file-backed `better-sqlite3` WAL, each slot uses one private guard database +beside the canonical wallet database, named `.snapshot-owner-.sqlite`. +The binding records both files' device/inode identity and a persistent random +guard marker. The source holds a guard write lock on the same physical connection +as its wallet read view; foreground WAL writers remain independent. The private +pool closes the still-open transaction before acknowledging cleanup. An ordinary +commit or rollback would release the guard too early. A failed native close keeps +the guard and ownership pending rather than claiming drainage. + +MySQL binds the actual server UUID, database and fixed slot to a named advisory +lock on the source connection. Its next transaction is read-only repeatable-read. +Physical cleanup waits for the native socket close event, not just mysql2's +earlier quit callback or the stream's `destroyed` flag. A different actual backend +refuses the binding. This is a same-server contract; load-balanced/PXC failover +requires separate qualification and cannot infer safety from a configured URL. + +Admission and cancellation run one bounded recovery flight per provider, visiting +at most eight expired or terminal owners in sequence. A matching proof connection +must acquire the same guard, fence the exact claim under the shared capacity lock, +close physically and then acknowledge that owner. An occupied guard leaves cleanup +pending. A still-unbound claim can be fenced under the capacity lock before a late +binder enters. Independent backend and cleanup errors remain observable. Provider +and service shutdown await an already-started recovery flight. + +An unproved native close keeps the provider's source slot fenced even after its +read promise settles. Further snapshot sources and recovery on that provider +refuse, and destruction preserves the cleanup error. An ordinary read failure +still permits a later source after physical cleanup succeeds. Keep the failed +owner reserved until backend recovery independently proves closure. + +Run migrations before capture, keep candidate binaries uniform, and drain all +captures before downgrade. Guard files are persistent coordination state: do not +unlink, replace or recreate a bound file to clear a reservation. Missing files, +changed identities and changed markers refuse recovery. This implementation +qualifies stable local filesystems only; database relocation/restoration and +shared/distributed filesystems require an explicit binding-transition procedure +and separate evidence. Standard wallet tables and BRC-38/39 bytes are unchanged. + +Synthetic tests cover eight simultaneous views, immutable reads with foreground +writes, cancellation/expiry, delayed acquisition and native closure, stale-owner +acknowledgements, missing/changed guard identity and cleanup failure. At least 300 +generated guarded schedules exercise recovery and successor ownership. The +existing native crash and MySQL fixture commands also terminate an owned child +process, verify the reservation before loss and recover it afterward while +preserving foreground writes. These fixtures do not establish deployed limits, +distributed filesystems, PXC or physical mobile behavior. Reader advertisement +remains disabled pending complete lifecycle and program qualification. + +## Internal journal foundation (unadvertised) + +The candidate includes internal SQL journal primitives for exact signed63 decimal +revisions, bounded composite metadata pages, thirteen-table observers, and +256-row bootstrap steps. Page limits charge every examined row. SQLite and MySQL +generations bind source schema and typed object ownership; MySQL persists an +atomic first intent and resumes each implicit-DDL step using the same epoch. +Unexpected objects or changed metadata refuse adoption. Standard source rows are +preserved by these helpers. Schema changes and concurrent migrators must be +excluded by the operator during installation or resumption. SQLite observer +source probes use native point-key equality plus an exact-byte residual, retaining +BINARY/NOCASE/RTRIM identity while admitting the complete source key index. + +Bootstrap callers supply an explicit row allowance from zero through 2,147,483,647. +The first copy transaction persists it; later pages require the same allowance. +Each page owns its transaction and refuses a caller transaction before accessing +the journal, preventing a retained writer barrier or an older MySQL progress view. +Each examined record consumes one unit, including a key already created by a live +observer, so the accounting does not depend on driver affected-row semantics. +Charges and progress commit atomically. An over-budget page inserts no metadata, +leaves the cursor and charge unchanged, and commits `capacity-exhausted` +invalidation; standard source writes remain available. Raising the allowance +requires a new owned generation. This bounds bootstrap allocations and does not +add quota accounting to ordinary MySQL source transactions. It does not bound +physical database/WAL bytes, event metadata or receipt retention. + +Native recovery fixtures include allowance binding, metadata insertion, progress +update and commit boundaries, then independently count the fully bootstrapped +streams to detect lost or repeated charges after recovery. + +Owned generations take an explicit receipt policy: 1–128 retained records and +a lifetime from 1 millisecond through 30 days. Installation snapshots this policy; +resume/read/completion refuse a mismatch. SQLite owns both tables and their expiry +index atomically; MySQL binds the policy into the durable installation plan and +validates each table's exact typed metadata, ownership and seed after implicit DDL. + +The internal receipt store binds backend, generation, schema/source, storage and +profile identity. Exact retries preserve the original high-water, expiry and +captured floor; a missing receipt refuses continuity even if newer writes have +surpassed its revision. MySQL decisions use current, nonwaiting locking reads, +including after an older transaction view. SQLite mutation transactions require +zero busy timeout. Expired records remain charged until collection commits; each +indexed collection removes at most 64 records. Stored text projections and +capacity probes are bounded. Receipt record/collection process-loss fixtures +check state before and after commit under both MySQL isolation levels. + +The internal `StorageKnex.openSnapshotJournalSource` controller reserves two +owned one-slot pools before taking the nonwaiting writer barrier. Fresh +`better-sqlite3` pools bind one existing file-backed WAL database; `mysql2` pools +bind the actual matching server UUID and database returned by both native +connections. Dynamic/external pools and other drivers refuse before construction. +The controller pins generation, profile, schema and all thirteen standard tables, +records a fresh request-bound prefix receipt, and awaits both commit and the +transaction's completion promise. Closure verification runs after the writer +barrier is released and before publication. Foreground writers can progress while +that verification or later reads run. + +Cancellation rejects opening promptly while the provider retains its source +admission until pending work and physical cleanup drain. Provider destruction +closes an active view. Unproved native cleanup permanently fences further source +admission and remains observable through cleanup and destruction. Native WAL and +MySQL fixtures terminate the source process before receipt commit, after durable +commit and after publication, then check receipt atomicity, new capture and writer +progress. MySQL additionally exercises READ COMMITTED and REPEATABLE READ, busy +barrier refusal and native provider shutdown. These are local synthetic engine +proofs; exact-head hosted, deployed/PXC and power-loss acceptance remain separate. + +Internal floor advancement reserves the exclusive global writer clock before +retention and receipt locks. A bounded full receipt scan (at most 129 rows) +validates every retained record and uses database-clock expiry. The floor is +monotonic and cannot pass either its fresh allocated prefix or any live receipt's +high-water mark. Expired receipts still occupy capacity until their separate +bounded collector commits. + +Internal tombstone collection scans at most 256 primary-key records per short +transaction, including live and newer rows before applying eligibility. Its +cursor binds the complete composite key, generation epoch, floor and stream; +changing the floor starts another pass. Exact revision/presence conditions guard +each deletion, and all selected metadata is validated first. MySQL current reads +use NOWAIT and explicit equality-prefix ranges; SQLite uses indexed tuple seeks +with exact UTF-8 order. The collector removes only old absent metadata, preserves +all thirteen source tables and does not promise physical file shrink. + +WAL and both MySQL isolation cohorts prove active-prefix pins, retained-view +immutability, revisions above the JavaScript safe-integer limit and bounded seek +plans. Twenty-four actual process-loss cuts cover floor commit, partial deletion, +collection commit and committed lost acknowledgements, then independently check +metadata/source preservation and writer progress. These remain synthetic engine +proofs, with production/PXC and power-loss acceptance separate. + +The internal `StorageKnex.maintainSnapshotJournal` controller owns each bounded +floor or collection transaction. It shares admission with retained capture, +validates the complete owned generation, ceiling and receipt policy, and excludes +the configured Knex migration owner before maintenance. SQLite reserves its WAL +writer before generation reads; MySQL locks only the configured migration-owner +row during schema validation, allowing foreground wallet writes until the short +clock/retention operation. Raw operator DDL remains outside this owner contract. + +Requests detach their cursor keys before configuration work. Cancellation or a +1–30,000 millisecond lifetime rejects the result promptly while physical drain +continues to own provider admission. Destruction fences admission synchronously; +transaction or pool cleanup failure remains observable and prevents new retained +work. A cancellation after commit can lose its result but cannot undo that commit; +recovery reads the durable floor/metadata before repeating a bounded operation. +WAL and actual RC/RR owned pools exercise eight process-loss cuts each, plus +provider cancellation, shutdown and physical native cleanup. + +Registered generation/retention recovery and generation-aware delta-page/receiver +integration remain unfinished. A durable prefix proof does not reopen a killed +database read transaction. + +This foundation adds no registered migration, public capability or reader +advertisement. Its event-window invalidation is not a complete retention or +resource policy. Runtime quotas and registered generation lifecycle, +generation-aware receiver/primary integration, and remaining remote/IndexedDB, +streaming and staged-import acceptance remain unfinished. Do not infer full +incremental continuity or completed issue #544 from these helpers. + +The wallet-snapshot-journal mutation target owns all twenty-two complete source +modules in twenty execution parts, including the whole receipt, collection, maintenance, capture, native +backend, connection ownership and migration/barrier modules. Every part retains the complete canonical +journal tests and fixtures, including one governed property entry for exact +revision/page, generated source-observer and committed receipt-state schedules. A minimum score of 90%, zero +uncovered/invalid/unexecuted mutants, 300 cases with seed 3242026, four workers, runner reuse 8 and 90-minute +bounds remain in force. Native ownership, client/server process-loss and broader +platform/performance evidence are separate required validation; source helpers +alone do not establish deployment, replication, PXC or power-loss readiness. + +Required wallet CI shard 1 runs the SQLite journal process-loss fixture and the +owned disposable MySQL client/server recovery fixtures after the existing archive +proofs, using the same-head build. The journal steps have three- and ten-minute +job-step bounds respectively. Each mutation part enforces zero uncovered and +invalid mutants; the complete disjoint aggregate must also reach 90%. A successful +part or a retried native worker does not establish complete-target qualification. + +### Postgres compatibility after main reconciliation + +Postgres uses its standard wallet schema and legacy synchronization path. The +SQLite/MySQL auxiliary snapshot migrations are recorded as no-ops on Postgres; +no retained or paged snapshot source is advertised. A future Postgres snapshot +implementation needs new forward migrations rather than changing those recorded +no-op entries. The bounded legacy transfer tables use Postgres `bytea`, and +settings validation accepts its `Postgres` database type. This preserves main's +storage contract while the #544 snapshot program remains incomplete. diff --git a/docs/packages/helpers/amountinator.md b/docs/packages/helpers/amountinator.md index c20057746..89d40da16 100644 --- a/docs/packages/helpers/amountinator.md +++ b/docs/packages/helpers/amountinator.md @@ -5,8 +5,8 @@ kind: package domain: helpers version: '2.1.7' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-08-26' -last_verified: '2026-08-26' +last_updated: '2026-09-23' +last_verified: '2026-09-23' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/amountinator' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/helpers/amountinator' @@ -16,6 +16,10 @@ tags: [helpers, amounts, satoshis] # @bsv/amountinator +The unpublished 2.1.7 candidate refreshes the packed first-party dependency +ranges for the next wallet interoperability release; no independent API or wire +format changes are introduced. Adopt after the dependency graph is published. + This source candidate declares SDK peer `^2.1.6 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/helpers/fund-wallet.md b/docs/packages/helpers/fund-wallet.md index 0c23c3664..830620d37 100644 --- a/docs/packages/helpers/fund-wallet.md +++ b/docs/packages/helpers/fund-wallet.md @@ -5,8 +5,8 @@ kind: package domain: helpers version: '1.5.3' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-08-31' -last_verified: '2026-08-31' +last_updated: '2026-09-23' +last_verified: '2026-09-23' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/fund-wallet' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/helpers/fund-wallet' @@ -16,6 +16,10 @@ tags: [helpers, testing, faucet, development] # @bsv/fund-wallet +The unpublished 1.5.3 candidate refreshes the packed first-party dependency +ranges for the next wallet interoperability release; no independent API or wire +format changes are introduced. Adopt after the dependency graph is published. + This source candidate declares SDK peer `^2.1.6 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/helpers/templates.md b/docs/packages/helpers/templates.md index b2d10776a..f80a28f37 100644 --- a/docs/packages/helpers/templates.md +++ b/docs/packages/helpers/templates.md @@ -46,6 +46,14 @@ const decodedData = OpReturn.decode(lockingScript) console.log(decodedData) // ['APP', '{"action":"vote"}'] ``` +## CommonJS compatibility + +The 1.10.3 candidate fixes SDK default-import double wrapping in CommonJS. +Both module formats retain the same root/wildcard exports and script encodings. +Packed acceptance constructs and signs scripts, including Mandala, multisig, +MultiPushDrop and R1K1 recovery, with published SDK 2.8.0 and the candidate SDK. +This source candidate is not yet published. + ## What it provides - **OpReturn** — Non-spendable data storage; create and decode OP_RETURN scripts diff --git a/docs/packages/helpers/wallet-helper.md b/docs/packages/helpers/wallet-helper.md index 95dba3726..224beeb3c 100644 --- a/docs/packages/helpers/wallet-helper.md +++ b/docs/packages/helpers/wallet-helper.md @@ -5,8 +5,8 @@ kind: package domain: helpers version: '0.1.9' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-08-27' -last_verified: '2026-08-27' +last_updated: '2026-09-23' +last_verified: '2026-09-23' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/wallet-helper' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/helpers/bsv-wallet-helper' @@ -16,6 +16,10 @@ tags: [helpers, wallet, transaction-builder] # @bsv/wallet-helper +The unpublished 0.1.9 candidate refreshes the packed first-party dependency +ranges for the next wallet interoperability release; no independent API or wire +format changes are introduced. Adopt after the dependency graph is published. + This source candidate declares SDK peer `^2.1.6 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/messaging/authsocket-client.md b/docs/packages/messaging/authsocket-client.md index f810420b3..64f884cc3 100644 --- a/docs/packages/messaging/authsocket-client.md +++ b/docs/packages/messaging/authsocket-client.md @@ -5,8 +5,8 @@ kind: package domain: messaging version: '2.1.8' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-08-26' -last_verified: '2026-08-26' +last_updated: '2026-09-24' +last_verified: '2026-09-24' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/authsocket-client' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/authsocket-client' @@ -22,6 +22,12 @@ before adopting it. > Client-side BRC-103 mutual authentication wrapper for socket.io-client. Signs all outbound messages and verifies inbound messages using a wallet, enabling authenticated peer-to-peer WebSocket communication. +## Next release candidate + +Unpublished 2.1.8 refreshes the UMD browser bundle with the integrated SDK. +The public API, signed event JSON and ESM/CommonJS SDK peer range are unchanged. +Existing wallet releases keep their published dependency pins. + ## Install ```bash diff --git a/docs/packages/messaging/message-box-client.md b/docs/packages/messaging/message-box-client.md index 6db6921ec..2c49d80ac 100644 --- a/docs/packages/messaging/message-box-client.md +++ b/docs/packages/messaging/message-box-client.md @@ -5,8 +5,8 @@ kind: package domain: messaging version: '2.6.0' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-09-25' -last_verified: '2026-09-25' +last_updated: '2026-09-28' +last_verified: '2026-09-28' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/message-box-client' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/message-box-client' @@ -16,6 +16,8 @@ tags: [messaging, message-box, brc-103, brc-29] # @bsv/message-box-client +The unpublished 2.6 candidate accepts strict bounded base64 transaction bytes at the PeerPay receive boundary, alongside deployed array/byte representations. Outgoing encoding stays unchanged. This does not remove encrypted relay message limits or provide a durable settlement journal. + This source candidate declares SDK peer `^2.8.6 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/middleware/402-pay.md b/docs/packages/middleware/402-pay.md index f15ae02f1..be3edd49e 100644 --- a/docs/packages/middleware/402-pay.md +++ b/docs/packages/middleware/402-pay.md @@ -5,8 +5,8 @@ kind: package domain: middleware version: '0.3.3' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-09-08' -last_verified: '2026-09-08' +last_updated: '2026-09-23' +last_verified: '2026-09-23' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/402-pay' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/402-pay' @@ -16,6 +16,8 @@ tags: [middleware, payment, '402', client] # @bsv/402-pay +The unpublished 0.3.3 candidate documents the BRC-121 adapter boundary. Its runtime wire contract is unchanged; BRC-118 support is provided by the SDK and composed auth/payment middleware, as described in the [payment transport guide](../../guides/brc118-payments.md). + This source candidate declares SDK peer `^2.1.6 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/middleware/auth-express-middleware.md b/docs/packages/middleware/auth-express-middleware.md index 8618069f2..07794cef1 100644 --- a/docs/packages/middleware/auth-express-middleware.md +++ b/docs/packages/middleware/auth-express-middleware.md @@ -3,10 +3,10 @@ id: pkg-auth-express-middleware title: '@bsv/auth-express-middleware' kind: package domain: middleware -version: '2.2.9' +version: '2.3.0' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-09-24' -last_verified: '2026-09-24' +last_updated: '2026-09-28' +last_verified: '2026-09-28' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/auth-express-middleware' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware' @@ -16,7 +16,9 @@ tags: [middleware, express, auth, brc-103, brc-104] # @bsv/auth-express-middleware -This source candidate declares SDK peer `^2.8.5 || ^3.0.0`. SDK3 remains +The unpublished 2.3 candidate adds opt-in bounded raw request capture for exact BRC-118 authentication. Mount it before body parsers; multipart extraction happens only after authentication. See the [BRC-118 deployment guide](../../guides/brc118-payments.md). + +This source candidate declares SDK peer `^2.9.0 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/middleware/payment-express-middleware.md b/docs/packages/middleware/payment-express-middleware.md index 1c1ea13f1..d10d569dd 100644 --- a/docs/packages/middleware/payment-express-middleware.md +++ b/docs/packages/middleware/payment-express-middleware.md @@ -3,10 +3,10 @@ id: pkg-payment-express-middleware title: '@bsv/payment-express-middleware' kind: package domain: middleware -version: '2.1.9' +version: '2.2.0' source_repo: 'bsv-blockchain/ts-stack' -last_updated: '2026-09-24' -last_verified: '2026-09-24' +last_updated: '2026-09-28' +last_verified: '2026-09-28' review_cadence_days: 30 npm: 'https://www.npmjs.com/package/@bsv/payment-express-middleware' repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/payment-express-middleware' @@ -16,7 +16,9 @@ tags: [middleware, express, payment, '402', brc-29] # @bsv/payment-express-middleware -This source candidate declares SDK peer `^2.1.6 || ^3.0.0`. SDK3 remains +The unpublished 2.2 candidate adds opt-in BRC-118 multipart payments behind verified raw authentication. It preserves the existing payment validation, replay and internalization path. See the [BRC-118 deployment guide](../../guides/brc118-payments.md), including CORS and receiver-first rollout. + +This source candidate declares SDK peer `^2.9.0 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/overlays/overlay-discovery-services.md b/docs/packages/overlays/overlay-discovery-services.md index d62ccb80a..780a48034 100644 --- a/docs/packages/overlays/overlay-discovery-services.md +++ b/docs/packages/overlays/overlay-discovery-services.md @@ -5,8 +5,8 @@ kind: package domain: overlays npm: '@bsv/overlay-discovery-services' version: '2.2.7' -last_updated: '2026-09-26' -last_verified: '2026-09-26' +last_updated: '2026-10-01' +last_verified: '2026-10-01' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services' status: stable @@ -15,6 +15,10 @@ tags: ['overlay', 'discovery'] # @bsv/overlay-discovery-services +The unpublished 2.2.7 candidate refreshes the packed first-party dependency +ranges for the next wallet interoperability release; no independent API or wire +format changes are introduced. Adopt after the dependency graph is published. + This source candidate declares SDK peer `^2.4.0 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/overlays/overlay-express.md b/docs/packages/overlays/overlay-express.md index 05fe5941c..7e5511f5a 100644 --- a/docs/packages/overlays/overlay-express.md +++ b/docs/packages/overlays/overlay-express.md @@ -5,8 +5,8 @@ kind: package domain: overlays npm: '@bsv/overlay-express' version: '2.7.4' -last_updated: '2026-09-26' -last_verified: '2026-09-26' +last_updated: '2026-10-01' +last_verified: '2026-10-01' review_cadence_days: 30 repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express' status: stable @@ -15,6 +15,10 @@ tags: ['overlay', 'express', 'http'] # @bsv/overlay-express +The unpublished 2.7.4 candidate refreshes the packed first-party dependency +ranges for the next wallet interoperability release; no independent API or wire +format changes are introduced. Adopt after the dependency graph is published. + This source candidate declares SDK peer `^2.4.0 || ^3.0.0`. SDK3 remains a coordinated proposal; see the [qualification and migration limits](../../guides/identity-did-vc-migration.md) before adopting it. diff --git a/docs/packages/overlays/overlay-topics.md b/docs/packages/overlays/overlay-topics.md index 0885d6cdd..ce67c2dce 100644 --- a/docs/packages/overlays/overlay-topics.md +++ b/docs/packages/overlays/overlay-topics.md @@ -47,6 +47,10 @@ These descriptive registries do not install executable permission modules. npm install @bsv/overlay-topics ``` +The unpublished 1.9.2 candidate refreshes the packed templates range for its +CommonJS compatibility repair. Topic APIs, schemas and admission behavior are +unchanged; no API migration is required. + ## Quick start ```typescript diff --git a/docs/packages/sdk/bsv-sdk.md b/docs/packages/sdk/bsv-sdk.md index 99234a64f..db674d02f 100644 --- a/docs/packages/sdk/bsv-sdk.md +++ b/docs/packages/sdk/bsv-sdk.md @@ -15,7 +15,14 @@ repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk' # @bsv/sdk -The isolated SDK3 retirement candidate removes the obsolete serial-DID token +The unpublished 3.0 candidate adds bounded BRC-118 payment transport to `AuthFetch` and corrects recipient-side BRC-29 derivation. See the [BRC-118 integration and migration guide](../../guides/brc118-payments.md) for preparation, negotiation, exact-byte authentication and uncertain-payment recovery. Existing nonempty non-multipart signing preimages and the 8 KiB header selection default are preserved. + +Ordinary HTTP fallback retains native text, URLSearchParams, typed Blob and +FormData serialization and inferred Content-Type, including cached peers. +Mutable inputs are copied before asynchronous work; explicit headers prevail. +Authenticated payment retries retain their owned-byte snapshots. + +The integrated SDK3 candidate removes the obsolete serial-DID token validation module. Native identity, certificate and authentication APIs remain. Use the [identity/DID/VC guide](../../guides/identity-did-vc.md) and [migration map](../../guides/identity-did-vc-migration.md). SDK3 consumer peer @@ -107,13 +114,13 @@ clients share the same regression coverage and a portable conformance vector. No API, wire or account-data migration is required; only clients already bundling the affected hardened SDK need a dependency update for this defect. -The 2.8.3 candidate repairs HTTP wallet discovery with an explicit BRC100 +The included 2.8.3 compatibility fixes repair HTTP wallet discovery with an explicit BRC100 originator and binds the default JSON transport fetch receiver for browsers. It restores signed `listActions` net amounts with the historical wire encoding, matching JSON validation; count, length and individual output bounds stay intact. Apps affected by these client defects can update the SDK without changing calls. Wallets retain the existing BRC100 contract; an ecosystem-wide application -migration is not required. The candidate is not published until the protected +migration is not required. The 3.0.0 candidate remains unpublished until its protected release workflow completes. The action-history compatibility regression affects signed `listActions` values diff --git a/docs/packages/wallet/wallet-toolbox-client.md b/docs/packages/wallet/wallet-toolbox-client.md index 578b7ad58..1c0e962b2 100644 --- a/docs/packages/wallet/wallet-toolbox-client.md +++ b/docs/packages/wallet/wallet-toolbox-client.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-client title: '@bsv/wallet-toolbox-client' kind: package domain: wallet -version: '2.14.5' +version: '2.15.0' last_updated: '2026-09-29' last_verified: '2026-09-29' review_cadence_days: 30 @@ -15,6 +15,8 @@ tags: [wallet, browser, indexeddb, storage, brc-100] # @bsv/wallet-toolbox-client +The unpublished 2.15 candidate exposes resumable sync with atomic IndexedDB page commits and foreground work between pages. See the [sync contract and rollout guide](../../guides/wallet-sync-reliability.md), including cancellation, inclusive timestamp boundaries and measured native-browser behavior. + `@bsv/wallet-toolbox-client` is the browser-safe Wallet Toolbox distribution. It includes the BRC-100 wallet, signer, services, IndexedDB storage, and remote storage client without Node-only Knex, SQLite, MySQL, or filesystem adapters. diff --git a/docs/packages/wallet/wallet-toolbox-mobile.md b/docs/packages/wallet/wallet-toolbox-mobile.md index 1a851519e..b3a8c9529 100644 --- a/docs/packages/wallet/wallet-toolbox-mobile.md +++ b/docs/packages/wallet/wallet-toolbox-mobile.md @@ -3,7 +3,7 @@ id: pkg-wallet-toolbox-mobile title: '@bsv/wallet-toolbox-mobile' kind: package domain: wallet -version: '2.14.5' +version: '2.15.0' last_updated: '2026-09-29' last_verified: '2026-09-29' review_cadence_days: 30 @@ -15,6 +15,8 @@ tags: [wallet, react-native, mobile, storage, brc-100] # @bsv/wallet-toolbox-mobile +The unpublished 2.15 candidate exports the portable resumable sync API. Remote destinations retain exclusive execution unless a future capability explicitly guarantees safe concurrency; this package does not acquire a local native database. See the [sync contract and rollout guide](../../guides/wallet-sync-reliability.md). + `@bsv/wallet-toolbox-mobile` is the React Native and mobile-safe Wallet Toolbox distribution. It includes wallet, signer, services, monitoring, and remote storage surfaces without Knex, SQLite/MySQL, IndexedDB, or Node-only IO. diff --git a/docs/packages/wallet/wallet-toolbox.md b/docs/packages/wallet/wallet-toolbox.md index acf8b7065..a55efd9b9 100644 --- a/docs/packages/wallet/wallet-toolbox.md +++ b/docs/packages/wallet/wallet-toolbox.md @@ -4,7 +4,7 @@ title: '@bsv/wallet-toolbox' kind: package domain: wallet npm: '@bsv/wallet-toolbox' -version: '2.14.5' +version: '2.15.0' last_updated: '2026-09-29' last_verified: '2026-09-29' review_cadence_days: 30 @@ -15,6 +15,8 @@ repo: 'https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wall # @bsv/wallet-toolbox +The unpublished 2.15 candidate adds `syncFromReaderResumable`, fair foreground scheduling, network/error preflight and canonical proof recovery. See the [sync contract and rollout guide](../../guides/wallet-sync-reliability.md). Existing sync methods and BRC-38/39/40 formats remain supported. + `@bsv/wallet-toolbox` is the reference toolkit for building BRC-100 wallets. It connects `@bsv/sdk` primitives to wallet storage, key derivation, signing, services, monitoring, permissions, and authentication flows. Use this package when you are building a wallet product, a wallet-like service, or another implementation that must match BRC-100 behavior. diff --git a/docs/reference/ci-performance.md b/docs/reference/ci-performance.md index 1c816a980..480212300 100644 --- a/docs/reference/ci-performance.md +++ b/docs/reference/ci-performance.md @@ -184,6 +184,12 @@ no privileged trigger or additional write/OIDC permission is granted. The upload processing check and final notifications run for every nonempty report, so the advisory `codecov/patch` report is available for contributors as well as maintainers. The repository-owned 90% patch-coverage check is mandatory on the exact diff. +TypeScript classification compares JavaScript emitted by the locked compiler. +A new declaration-only file is compared with empty source only after Git's base +tree proves the path was absent. Runtime code, enums and side-effect imports +remain governed; unsupported syntax remains governed, and Git/read errors fail +the gate. This avoids an impossible LCOV obligation for erased declarations +without adding path exclusions or reducing the coverage floor. The repository ruleset must require `merge-gate` and must not require the duplicate external `codecov/patch` status. Uploading, processing and notification run in a separate reporting job after the local gate, so an external service outage cannot @@ -341,3 +347,31 @@ This source proof cannot qualify a newer wallet head: its owner must compare the full pinned inventory/configuration on the final source before adoption. Registry, runtime, assertions, workers, deadlines and thresholds remain owned and unchanged by the partition facility. + +The SQLite conflict-repair extension retains all eight canonical helper files in +five whole-file groups: identity, membership, generation/legacy ownership, +bootstrap/state and retirement/migration. Identity and membership originally +shared a part. On source `18748e2f4`, that part passed all 759 baseline tests and +instrumented 465 mutants, but reached its 90-minute limit without a complete +report; three native worker crashes remain in the failed-run evidence. Separating +the two complete files preserves every canonical source, test and fixture, four +workers, runner reuse of eight, the 90-minute per-part limit, and each critical +score/uncovered/invalid gate. Complete fresh qualification is required; splitting +does not establish a runtime improvement or turn the timed-out run into a pass. + +### Remote reader execution allowance + +The complete `wallet-snapshot-remote-reader` campaign at wallet source +`de75e1d5150f5f0dda4e81e6631e55c17d5f8a30` passed in 2,691.70 seconds: +1,248 mutants, 943 killed, 235 timed out and 70 survived, with zero uncovered +or invalid outcomes and a 94.39% global score. All 785 source/test files and +13 configuration inputs stayed fixed. Worker exits were recovered by the +runner and remain in the raw log; their native cause is unconfirmed. + +That local execution leaves only 8.30 seconds of the former 45-minute hosted +job allowance for installation, build restoration and reporting. The reader +therefore joins the existing bounded 90-minute allowance in both workflows. +All prior allowances and the 45-minute default remain; source/test unions, +property settings, workers and the original global gates are unchanged. Record +actual hosted setup/execution overhead before drawing performance conclusions. +This is an execution allowance, not a measured speedup or full #544 acceptance. diff --git a/docs/reference/container-supply-chain.md b/docs/reference/container-supply-chain.md index 4cda0a7b3..feb86710b 100644 --- a/docs/reference/container-supply-chain.md +++ b/docs/reference/container-supply-chain.md @@ -61,6 +61,21 @@ Repository health requires every final runtime stage to install every active pin. A base refresh that incorporates the fix must remove the obsolete runtime pins and their Dockerfile installs together. +The September 30, 2026 source refresh advances `libcrypto3` and `libssl3` to +`3.5.9-r0` across all seven images. Alpine replaced the previously pinned +`3.5.8-r0` packages in its rolling stable repository, causing exact-version +installs to fail. The reviewed [OpenSSL 3.5.9 security release](https://openssl-library.org/news/openssl-3.5-notes/) +retains the earlier CVE-2026-14456 fix and adds the September security fixes, +including CVE-2026-84782. The official [Alpine v3.24 x86_64 package index](https://dl-cdn.alpinelinux.org/alpine/v3.24/main/x86_64/APKINDEX.tar.gz) +retrieved at review names both packages at `3.5.9-r0` from aports commit +`29b9ec24b1b5b39aeef51fa2a044210e2ec5258e`; `libssl3` requires that exact +`libcrypto3` version. Both retain their `.so.3` ABI. This patch changes neither +the immutable Node base nor service APIs, configuration or persistence. +All seven Linux/amd64 build, scan and runtime gates must validate the source +candidate; this record does not claim a publication or deployment. A deployed +rollback uses a previously verified complete image digest; it must not restore +the vulnerable base libraries by removing the pins. + Package locks under `infra/**/package-lock.json` are committed release inputs. Release workflows never rewrite them. A stale or inconsistent lock therefore fails `npm ci` before publication instead of silently producing a different diff --git a/docs/reference/dependency-policy.md b/docs/reference/dependency-policy.md index 4166d632e..0b1191734 100644 --- a/docs/reference/dependency-policy.md +++ b/docs/reference/dependency-policy.md @@ -2,9 +2,9 @@ id: dependency-release-policy title: 'Dependency and Release Policy' kind: reference -version: '1.3.2' -last_updated: '2026-09-23' -last_verified: '2026-09-23' +version: '1.3.4' +last_updated: '2026-10-02' +last_verified: '2026-10-02' review_cadence_days: 30 status: stable tags: [reference, dependencies, security, releases] @@ -108,6 +108,60 @@ notes and necessity, runtime/build/peer compatibility, lockfile deduplication, audit and CodeQL results, package and consumer tests, bundle/performance impact, and affected public versions. +The docs-site Mermaid graph selects DOMPurify 3.4.16 instead of 3.4.13 for +[GHSA-p98j-92pf-mc4p](https://github.com/advisories/GHSA-p98j-92pf-mc4p). +The reviewed [3.4.16 release](https://github.com/cure53/DOMPurify/releases/tag/3.4.16) +fits Mermaid 11.16.1's existing `^3.3.3` range and exceeds the seven-day release +age. pnpm regenerates only that package resolution, integrity and existing graph +reference; all importers, manifests and unrelated resolutions remain unchanged. +The governed bundle inventory records the same version, with its license +expression and license-file hash unchanged. This private documentation dependency does not change wallet runtime packages, +public APIs or candidate versions. Frozen installation, root checks, docs tests +and a built-site browser check qualify the ordinary Mermaid consumer. No service +or package is deployed by this source change. + +## Temporary Metro watcher dependency repair + +Metro-file-map 0.87.1 uses only micromatch.some(), whose matcher is already +Picomatch 2.3.2. The exact-version paired source/distribution patch calls that +same loop directly, declares the same exact Picomatch dependency and removes +only the scoped micromatch dependency and its now-unused braces closure. +The reviewed [braces advisory](https://github.com/advisories/GHSA-vfj7-8cjw-p6xm) +lists no patched release; the [upstream proposal](https://github.com/micromatch/braces/pull/72) +is still unreleased. No advisory exclusion or audit threshold change is added. + +The existing mobile platform gate checks 1,120 watcher results frozen from the +unmodified published implementation, event/stat/path contracts, and absence of +the removed matcher packages before its packed Metro/Hermes compilation, +source-map/composition and unchanged bundle budgets. The dated override registry +owns the patch, exact package extension and scoped removal as one repair. Remove +all three together when an official compatible release removes the affected +path and the full compatibility, frozen graph, audit and platform checks pass. +This build-tool repair does not change published wallet APIs or package versions. + +## Temporary standalone service watcher repair + +The basic and cloud UHRP services and WAB use Nodemon 3.1.14 with a parent-scoped +Chokidar 4.0.3 substitution, removing the same affected braces closure without +an advisory exclusion. Nodemon already resolves watched globs to literal +directories; the source-owned adapter retains the old anymatch 3 ignore +semantics, including recursive literal directories, custom cwd, dotfiles, +regex and function options. WAB replaces ts-node-dev with this same CLI. +The existing Node/ts-node/telemetry execution paths remain intact, with explicit +TypeScript and env extensions, manual `rs` and graceful shutdown regressions +run before each service's ordinary tests. The service-copy policy keeps both +adapter and native regression identical across all three service contexts. + +The adapter normalizes every backslash and repeated slash without changing its +ignore matching. Native watcher polling permits 400 predicate attempts, with a +25 ms wait after every failed attempt, and preserves early process-exit errors. + +The three new registered substitutions bring the combined retained count to 30. Remove them and the adapter together after a compatible official Nodemon +release resolves the dependency path natively and all watcher, frozen audit, +service and protected Linux image gates pass. These standalone development +tools change no public npm candidate version, service HTTP API, production +startup or persisted schema; deployed images require separate promotion. + ## Supply-chain controls `pnpm-workspace.yaml` is the source of truth for installation controls: @@ -136,13 +190,67 @@ must be a dependency or peer, and clean packed consumers must typecheck it. This keeps build-only advisory trees out of consumer installs without shipping unresolvable public declarations. -The root workspace carries seven narrow audited dependency overrides: - -- Jest 30.4.2 and Stryker still constrain parts of their reporting and coverage - graphs to minimatch releases with older `brace-expansion` ranges. - GHSA-rgw5-rvv9-x895 required 5.0.9. GHSA-6j4f-fj2g-mc7p requires 5.0.10 and - GHSA-qhr7-859c-m2p7 requires 5.0.11, so the workspace substitutes 5.0.11 - until every supported path resolves it natively. +The September 30 audit refresh also selects compatible transitive releases within +existing ranges: `engine.io` 6.6.10 in the workspace and Message Box server; +`ip-address` 10.7.1 in Message Box, both UHRP servers and WAB; `fast-uri` 3.1.8 in +Message Box; and `undici` 6.28.1 in WAB. These generated lock changes add no +manifest ranges or overrides. All selected versions exceed the seven-day release +age. Upstream releases preserve their module and Node contracts; service suites, +AuthSocket coverage and frozen-install audits qualify the affected consumers. +The relevant upstream releases are [Engine.IO](https://github.com/socketio/socket.io/releases/tag/engine.io%406.6.10), +[ip-address](https://github.com/beaugunderson/ip-address/releases/tag/v10.7.1), +[fast-uri](https://github.com/fastify/fast-uri/releases/tag/v3.1.8) and +[Undici](https://github.com/nodejs/undici/releases/tag/v6.28.1). +Source reconciliation does not release packages or deploy service images. + +The follow-up Axios audit correction selects `axios` 1.20.0 in the five existing +Message Box, UHRP basic, UHRP cloud, notifier and WAB locks. Only Axios resolution +and dependency metadata change; manifests, declared ranges, lock formats and all +other resolved versions are preserved. The already-selected `form-data` 4.0.6 +satisfies the raised dependency floor. Review of the +[1.19.0](https://github.com/axios/axios/releases/tag/v1.19.0) and +[1.20.0](https://github.com/axios/axios/releases/tag/v1.20.0) releases includes +configuration hardening, proxy behavior, cancellation and declaration changes. +The services use ordinary own-property request options; their frozen installs, +high/critical audits, builds where applicable and deterministic suites pass on +Node 24. Public workspace manifests and browser/mobile dependency graphs are +unchanged. Infrastructure locks are container/function build inputs, so this +correction needs no public npm version change or consumer migration. Hosted +Linux image and exact-head analysis gates remain required before promotion; +these local results do not establish deployed behavior. + +The subsequent gRPC correction selects `@grpc/grpc-js` 1.14.5 in all seven +standalone service locks for the September 30 audit correction, addressing +[GHSA-m9gg-hp2v-232j](https://github.com/advisories/GHSA-m9gg-hp2v-232j) and +[GHSA-f596-whhp-79r4](https://github.com/advisories/GHSA-f596-whhp-79r4). +The reviewed [1.14.5 release](https://github.com/grpc/grpc-node/releases/tag/%40grpc/grpc-js%401.14.5) +preserves the Node engine and dependency ranges; its fixes also cover stale call +retention, status fields and completed HTTP/2 streams. Chaintracks Server and +Wallet Infra additionally adopt the already-reviewed `ip-address` 10.7.2 within +their existing ranges; the [10.7.2 release](https://github.com/beaugunderson/ip-address/releases/tag/v10.7.2) accepts case-insensitive ARPA suffixes. Package-manager regeneration changes only these nine +resolved nodes; manifests, lock formats, all other dependencies, the workspace +lock and the earlier Axios correction are unchanged. + +All seven frozen installs, allowlisted native rebuilds, builds and lints pass +on Node 24, with 658 service tests and seven zero-finding audits. Wallet Infra +has no standalone test script; its build/lint and the shared dependency checks +are reported separately. Each installed gRPC copy passes an ordinary loopback +unary call and client cancellation; both changed IP consumers pass IPv4/IPv6 +parsing and invalid-syntax checks. The gRPC registry's unpacked size grows by +49,814 bytes; no public workspace/browser/mobile graph changes. These are +compatibility checks, not a throughput or memory benchmark. No public npm +version or consumer migration changes; protected Linux image and exact-head +analysis gates still qualify the eventual service artifacts before promotion. + +The combined workspace and standalone registries carry 30 audited dependency overrides, including: + +- Jest 30.5.1 and Stryker still constrain parts of their reporting and coverage graphs to + minimatch releases with older `brace-expansion` ranges. The follow-up + advisories GHSA-qhr7-859c-m2p7, GHSA-6j4f-fj2g-mc7p and + GHSA-q2hr-2g5m-vwhr require `brace-expansion` 5.0.12, so the workspace + and existing standalone substitutions select 5.0.12 until every supported + path resolves it natively. The release preserves the existing module exports, + types and Node engine range. - `socket.io` in `@bsv/authsocket` still admits `engine.io` releases below 6.6.10. GHSA-2gc4-cqfq-p2gv is a protocol-revision mismatch that can crash the Node process during a transport upgrade, so the workspace selects @@ -176,17 +284,15 @@ changed, stale, unowned, or upstream-unlinked override. Elapsed review dates produce maintenance reminders in source CI and fail the separate weekly maintenance audit (`node scripts/repository-health.mjs --maintenance`). -The 2026-10-01 review rechecked all 25 selectors against the frozen graph. -Jest, Stryker, and socket.io still admit the vulnerable `brace-expansion` and -`engine.io` ranges, so those two substitutions stay. The earlier findings still -hold for the rest: the supported graphs select exact `gaxios@7.1.3`, admit -`uuid@9`, and pin `qs@6.15.1` without their registered substitutions, while the -current frontmatter plugin still requests TOML 3.x. New upstream majors can -remove some legacy paths only through a coordinated Stryker or Google Cloud -migration. Metro 0.87.1 replaced its `image-size` dependency with an in-tree -parser, so the former Metro-scoped substitution was retired; the mobile -platform contract verifies that boundary. The method, result, count, and next -rehearsal are enforced in +The 2026-10-02 reconciliation retains 26 selectors from the reviewed main +updates, including the Mermaid/chevrotain `lodash-es` remediation and the +Yamux-scoped stream-interface compatibility substitution. It preserves the +qualified `brace-expansion` 5.0.12 workspace and standalone floor for +GHSA-q2hr-2g5m-vwhr; Jest and Stryker still admit vulnerable earlier ranges. +The `engine.io` substitution remains. Metro 0.87.1 replaced its `image-size` +dependency with an in-tree parser, so the former Metro-scoped substitution +was retired; the mobile platform contract verifies that boundary. The method, +result, count and next rehearsal are enforced in `governance/dependency-release-policy.json`. The independently locked OpenAPI generator also carries a narrow Redocly diff --git a/docs/reference/package-api-migrations.md b/docs/reference/package-api-migrations.md index e0f3e45c8..305a979fc 100644 --- a/docs/reference/package-api-migrations.md +++ b/docs/reference/package-api-migrations.md @@ -23,41 +23,41 @@ and clean-consumer tests remain the executable type authority. ## Current release boundary -| Package | npm baseline | Source | Candidate | API | Migration | -| --------------------------------- | ------------ | -------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `@bsv/402-pay` | `0.3.2` | `0.3.3` | patch | [API and usage](../packages/middleware/402-pay.md) | Custom clients must send a strictly framed BRC-95 Atomic BEEF envelope whose subject is the payment transaction; plain BEEF is no longer accepted. Legacy Atomic BEEF containing unrelated branches remains compatible because the server reduces it to the declared subject and dependency closure. The default replay store is bounded and process-local; production services with more than one serving process or node must inject the same durable atomic PaymentReplayStore everywhere. Low-level validators should retain one wallet object or pass an explicit store. Applications that relied on implicit console diagnostics must supply the optional structured logger. Treat an unchallenged 503 after payment submission as ambiguous and reconcile the transaction before requesting another payment. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/air-gap` | `0.1.3` | `0.1.3` | none | [API and usage](../packages/helpers/air-gap.md) | No migration is required for valid BRC-141 v1 senders or decoders. Malformed, oversized, duplicate-heavy, or adversarial high-degree sessions now fail closed or are evicted within the documented limits instead of consuming unbounded resources. | -| `@bsv/amountinator` | `2.1.6` | `2.1.7` | patch | [API and usage](../packages/helpers/amountinator.md) | Valid finite inputs retain the public API. Currency identifiers are trimmed and normalized to uppercase; callers that passed non-finite values, empty currencies, coercive options, invalid decimal precision, or non-finite converter results must normalize or reject them before calling because those values now fail closed. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/auth` | `0.1.5` | `0.1.6` | patch | [API and usage](../packages/middleware/auth.md) | No valid proof bytes or public API shape changes. Client and server must use the same protocol tuple; security levels 0, 1, and 2 remain supported and select wallet consent policy while the explicit counterparty scopes derivation at every level. Proof and wallet adapters must supply plain own data fields and dense byte arrays; inherited, accessor-backed, sparse, or malformed runtime shapes fail closed. Structured bodies containing non-finite numbers or negative zero must be normalized to an unambiguous wire representation before signing. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/auth-express-middleware` | `2.2.8` | `2.2.9` | patch | [API and usage](../packages/middleware/auth-express-middleware.md) | No BRC100 call, wire or wallet-data migration. Configure and validate transport header limits at the HTTP server, CDN, proxy or WAF; maxRequestBytes no longer caps received headers. Clients needing larger signed responses require corresponding client capacity, including SDK 2.8.5. Published 2.2.8 is verified against protected release 36052862859 and its immutable source artifacts. Upgrade the required SDK peer to 2.8.5; custom ExpressTransport/Peer setups should pass maxGeneralPayloadBytes=null when the HTTP layer owns payload capacity. SDK consumers may retain ^2.8.5 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/authsocket` | `2.1.8` | `2.1.9` | patch | [API and usage](../packages/messaging/authsocket.md) | No API or wire migration is required for valid JSON events. Existing numeric-key objects under byte-like names are unchanged; typed payment protocols recover historical byte objects at their explicit fields. Outbound non-JSON or ambiguous runtime values, including negative zero, nested undefined, sparse arrays, accessors, hidden/extra properties, and serialization hooks, now fail closed before signing. requestedCertificates is an SDK allowlist, not an application authorization verdict. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/authsocket-client` | `2.1.7` | `2.1.8` | patch | [API and usage](../packages/messaging/authsocket-client.md) | No API or wire migration is required for valid JSON events. Existing numeric-key objects under byte-like names are unchanged; typed payment protocols recover historical byte objects at their explicit fields. Outbound non-JSON or ambiguous runtime values, including negative zero, nested undefined, sparse arrays, accessors, hidden/extra properties, and serialization hooks, now fail closed before signing. The connect event/connected property report Socket.IO transport state, not completed BRC-103 authentication; wait for verified application traffic when local behavior requires a known unpinned server. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/btms` | `1.2.3` | `1.2.4` | patch | [API and usage](../packages/wallet/btms.md) | Existing local, mainnet, testnet, and number-array behavior is unchanged. TTN consumers select networkPreset teratestnet; all consumers should upgrade to @bsv/sdk 2.4.1 or later for byte-boundary compatibility. Valid canonical token amounts remain compatible. Audit historical tokens for signed, exponent, leading-zero, non-positive, or greater-than-9007199254740991 amount fields before rebuilding wallet state; public number-based operations now fail closed when an aggregate cannot be represented exactly. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/btms-permission-module` | `1.2.1` | `1.2.2` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | Valid canonical BTMS flows remain compatible. Custom prompts must return literal true, authorization maps may contain at most 1,024 entries, and callers must provide dense signatures, safe exact totals, and canonical preimages; coercive or trailing-byte forms now fail closed. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/chirp` | `0.1.3` | `0.1.4` | patch | [API and usage](../packages/network/chirp.md) | No API, codec, object identifier, service schema, or wire-format migration. Use @bsv/chirp 0.1.3 with SDK 2.8.9 or later and Cloud Bucket 0.2.48 or Lite 0.1.45 for authenticated binary uploads, including full 4 MiB chunks. Custom authenticated transports must send the supplied owned byte body unchanged and supply ordinary HTTP framing; Content-Type remains signed through AuthFetch. Earlier service entrypoints do not parse staged binary bytes before authentication and require this service patch. Existing stored objects, sessions, advertisements and leases remain unchanged. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/did` | `0.2.6` | `0.3.0` | minor | [API and usage](../packages/helpers/did.md) | DID0.3 is a breaking pre-1.0 candidate: compressed identity-key input only; remote document/lifecycle options rejected. Use original signed BRC52 binary for envelopes, or explicitly verified compatible structured core. Wire authenticated BRC103/104 payload/control, durable atomic replay, consent, issuer/schema/purpose reliance and local status policy; callbacks alone are not full transport conformance. See docs/guides/identity-did-vc-migration.md. Proposed custom W3C mechanisms remain unregistered. Import BRC52 APIs/types from @bsv/did/brc52; identity-key DID and independent SD-JWT helpers stay at @bsv/did. | -| `@bsv/ecpm-permission-module` | `0.1.1` | `0.1.2` | patch | [API and usage](../packages/wallet/ecpm-permission-module.md) | Valid ecpm requests remain compatible. Authorization callbacks must return literal true; inherited, accessor-backed, malformed, or oversized inputs now fail closed. Hosts must keep grants below 1,024 entries and avoid more than 64 concurrently pending new authorization prompts per module instance. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/fund-wallet` | `1.5.2` | `1.5.3` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No public API migration is required for conforming wallets. Custom WalletInterface adapters must return the literal accepted: true verdict after successful internalization; refusals and malformed or coercive results now throw instead of being reported as success. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/gasp` | `1.3.7` | `1.3.8` | patch | [API and usage](../packages/overlays/gasp.md) | No API or wire migration is required. In bidirectional mode, the receiver must already have the parent of a pushed child, request it during a subsequent synchronization round, or reject the graph because GASP v1 submitNode does not transmit the spentBy parent outpoint. Pull-only operation avoids that assumption. Custom storage and remote adapters must preserve exact request/node binding and bounded responses. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/lch` | `0.2.0` | `0.2.1` | patch | [API and usage](../packages/content/lch.md) | Replace buyer.quote(endpoint, request, issuer, keyGrants) with buyer.quote(verifiedSignedOffer, request, expectedSeller, keyGrants). JavaScript callers that still pass an endpoint string fail before transport I/O. Persist the returned plan and every partial settlement proof, retry with the same funded transaction, and configure a profile-aware agreementEvaluator before completion or recovery. Low-level unverified recovery results must not authorize key storage or content access. DNS endpoints require an address-pinning connector outside trusted browser environments. Distributors must retain THIRD_PARTY_NOTICES.md with the package; published BRC-170 remains authoritative if the implementation and standard differ. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/message-box-client` | `2.5.4` | `2.6.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | No migration required for the new fields: the PeerMessage paymentOutcome and retained payment are optional, and messages without a payment are unchanged. To avoid losing payments, store any returned payment (validate it first; it is the raw envelope value) before acknowledging its message. Install @bsv/sdk 2.8.6 or later alongside this package; payments refused on earlier SDKs remain pending in the message box and can be accepted after upgrading. No BRC100 wire or wallet-data change. SDK consumers may retain ^2.8.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/overlay` | `2.6.2` | `2.6.3` | patch | [API and usage](../packages/overlays/overlay.md) | No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/overlay-discovery-services` | `2.2.6` | `2.2.7` | patch | [API and usage](../packages/overlays/overlay-discovery-services.md) | No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/overlay-express` | `2.7.3` | `2.7.4` | patch | [API and usage](../packages/overlays/overlay-express.md) | No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/overlay-topics` | `1.9.1` | `2.0.0` | major | [API and usage](../packages/overlays/overlay-topics.md) | Topics2 is a breaking retirement candidate. Remove DIDTopicManager/createDIDLookupService/DIDRecord/DIDQuery and tm_did/ls_did usage. Resolve immutable identity-key DIDs offline; explicitly configured tm_identity/ls_identity supplies certificate discovery, not DID-document mutation or automatic trust. Preserve existing database/history and plan service rollout without automatic replacement installation. See docs/guides/identity-did-vc-migration.md. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. Mandala is a clean break with no data migration: the old MandalaToken/MandalaAdmin wire format is no longer admitted and 1.x Mandala rows are not read or converted, so start Mandala on a new database with new deploys (existing on-chain outputs are not spent or deleted). MandalaTopicManager drops adminWallet, adminProtocolID and stateStore.isAdminOutpoint and requires trustedIssuers (non-empty compressed lowercase public keys) and engineOutputs, with optional membership, membershipExempt and onOwnerRepair; pass the same MandalaStorageManager (a MandalaStateStore) to admission and lookup. Refusals are MandalaReject { code, reason } from the Reasons catalog; do not match on old error text. MandalaLinkagePayload is replaced by MandalaEnvelope with encodeEnvelope/decodeEnvelope; the register action is replaced by a deploy at output 0 with a deploySig. foldAction, defaultAssetState, AssetAdminState, FoldContext and MandalaTokenRecord use tokenId (_0) instead of assetId, drop issuerIdentityKey and add feeRatePerKb. Persisted schema (section 6.6): new mandalaOwners and mandalaAuthorities collections; mandalaTokens, mandalaMetadata, mandalaAssetStates and mandalaAdminHistory are keyed by tokenId, metadata holds the decoded deploy payload, and history rows store kind, detailsHex, commitment and delta (plus optional frozenAmount/frozenOwner on freezes). De-trusting an issuer key revokes the authority coins it holds; rotate keys by moving authority coins first. Operators register tm_mandala_registry/ls_mandala_registry explicitly. The infra/overlay-server wiring follows after publication (P1b). | -| `@bsv/paymail` | `2.4.9` | `2.4.10` | patch | [API and usage](../packages/messaging/paymail.md) | None. ESM consumers are unaffected. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/payment-express-middleware` | `2.1.8` | `2.1.9` | patch | [API and usage](../packages/middleware/payment-express-middleware.md) | No BRC100 call, wire or wallet-data migration. maxPaymentHeaderBytes is now ignored, including existing explicit values: move transport policy to the HTTP server, CDN, proxy or WAF and remove the deprecated option. Validate the complete HTTP route for supported payment proofs. Published 2.1.8 is verified against protected release 36052862859 and its immutable source artifacts. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/sdk` | `2.8.10` | `3.0.0` | major | [API and usage](../packages/sdk/bsv-sdk.md) | SDK3 is a separate breaking retirement candidate: remove identity/DIDTokenValidation imports and use @bsv/did BRC202 identity-key resolution plus optional @bsv/did/brc52 original-signature envelopes. Retain historical data and recover authenticated identity bindings; do not relabel serial DIDs. Existing acquisition-result validation fixes remain. First-party SDK3 peer alternatives require complete packed qualification and owner coordination; preserved SDK2 ranges are not silently narrowed. See docs/guides/identity-did-vc-migration.md. | -| `@bsv/simple` | `0.6.0` | `0.7.0` | minor | [API and usage](../packages/helpers/simple.md) | Simple0.7 is a breaking pre-1.0 candidate. Remove mutable DID/resolver/provider configuration, proxy handlers and DIDError/types; use DID.fromIdentityKey/resolve and wallet.getDID. CredentialIssuer.issue returns {credential,keyringForSubject}; verify accepts exact JSON or UTF8 bytes and returns structured cryptographic results. The HTTP verify route now requires {credential: originalEnvelopeJson} with original text as a string; parsed objects return HTTP400. Web request decoding rejects duplicate members and invalid UTF8, and custom parsed-body adapters must establish equivalent strict decoding. Preserve received text and inspect verification.verified, not HTTP success. Local revocation record status is retained/unknown, not inferred chain revocation. See docs/guides/identity-did-vc-migration.md. Replace createServerWalletHandler() deployments with createServerWalletHandler({ authorize: async ({ action, headers }) => authenticatedSessionCanUseAction(headers, action) }). The callback must return literal true for each status, create, request, receive, balance, outputs, or reset action; omission now returns HTTP 403 for every action. Roll out the authentication layer and callback with the package, update anonymous probes or automation, and apply the same policy to every replica. Do not emulate the old public behavior with an unconditional authorize: () => true callback. Valid recipient derivations and authenticated Message Box peers remain supported; malformed, wrong-owner, or transaction-mutated flows now fail closed. New CredentialSchema and Certifier records use canonical 32-byte types. Current SDK wallet methods reject historical short types, so do not put migration aliases in wallet list, acquire, prove, or relinquish calls. Export affected records through the storage version that created them, authenticate them offline against the exact locally configured identifier, and reissue/import canonical replacements; no legacy certificate is rewritten automatically. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. Persisted local field ordering requires no migration; transport adapters preserve original certificate signing bytes. | -| `@bsv/templates` | `1.10.2` | `2.0.0` | major | [API and usage](../packages/helpers/templates.md) | Breaking. Removed exports: MandalaToken, MandalaTokenDecoded, MandalaAdmin, ADMIN_PROTOCOL, MandalaAdminDecoded, MandalaActionDetails, MandalaActionKind, MandalaAdminLockParams, MandalaAdminUnlockParams and AssetMetadata. Removed subpath @bsv/templates/mandala-signing.ts. The @bsv/templates/mandala-encoding.ts subpath keeps only createMinimallyEncodedScriptChunk, decodeScriptNum and decodeScriptNumChunk; encodeScriptNum, encodeAssetId and decodeAssetId are removed. Replace MandalaToken/MandalaAdmin with Bsv21Binary: deploy, authority and value outputs are lock(tokenId \| null, amount, pubKeyHash, payload?) with bigint amounts and _0 token ids (tokenIdFromString/tokenIdToString), decoded with Bsv21Binary.decode. Mandala admin actions are committed by sha256 of strict-CBOR details in the authority output payload, not by a commitment-derived key. This is a clean break with no data migration: tokens and admin chains in the old format are not readable by 2.0.0. @bsv/overlay-topics 2.0.0 consumes this format; the infra/overlay-server wiring follows after publication. SDK consumers may retain ^2.1.6 or install the SDK3 candidate; SDK3 removes the obsolete DID token API (see docs/guides/identity-did-vc-migration.md). | -| `@bsv/teranode-listener` | `1.1.6` | `1.1.7` | patch | [API and usage](../packages/network/teranode-listener.md) | No API migration is required for valid consumers: raw callbacks remain the default and decoding is opt-in with decodeMessages: true. Configuration arrays and callbacks are snapshotted at construction, boolean controls must be literal booleans, and malformed or duplicate topics, addresses, keys, and unsupported properties now fail closed. usePrivateDHT: false now actually omits the DHT service. The published mainnet PNET value is transport compatibility data, not a publisher credential; decoded sender and payload fields remain untrusted and security-critical claims require independent validation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/verifast` | `0.3.6` | `0.3.7` | patch | [API and usage](../packages/sdk/verifast.md) | Valid typed verification calls and worker protocols remain compatible. Custom module factories and WASM adapters must return the exact documented binary and boolean shapes; coercive network, height, flag, byte, batch, verdict, lifecycle, or disposed-instance values now fail closed. Keep THIRD_PARTY_NOTICES.md and LICENSES/ with every JavaScript and WebAssembly distribution. SDK consumers may retain ^2.1.8 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/wallet-helper` | `0.1.8` | `0.1.9` | patch | [API and usage](../packages/helpers/wallet-helper.md) | getAddress now derives with forSelf: true and returns the caller-owned side of the bilateral relationship. Applications that stored or coordinated the previous peer-owned result must regenerate and exchange the corrected address before sending value. Amount must be an integer from 1 through 1,000 and counterparties must be valid public keys. Valid canonical transaction-builder flows remain supported; malformed, ambiguous, or wallet-mutated results now fail closed. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/wallet-relay` | `0.5.1` | `0.5.2` | patch | [API and usage](../packages/wallet/wallet-relay.md) | Enable signed QR codes and distribute only HTTPS pairing origins and relay API URLs. Root-relative API paths and loopback HTTP remain supported. Configure onApprovalRequired for every method not deliberately listed in autoApproveMethods; unsigned pairing URIs and implicit approval are no longer accepted. Existing relay sessions, custom RPC method names, and supported wallet RPC byte encodings remain valid, and host applications continue to provide their matching Express runtime and type graph. Version 0.5 defaults to two missed pongs; set maxMissedHeartbeats to 1 to retain the prior heartbeat policy. Heartbeat intervals must fit the Node timer range. Preserve Cache-Control: no-store at proxies, prefer the stable bsv-wallet-relay plus token WebSocket subprotocols over the legacy token query parameter, and configure edge rate limiting in addition to the bounded in-process defaults. The 24-hour connected-session lifetime no longer renews on reconnect, malformed wallet calls fail with code 400, and WalletRelayClient retains at most 100 log entries unless maxLogEntries is configured. The QR remains a short-lived bearer invitation: keep it private and require explicit operation approval. Relay envelopes, cryptographic framing, and wallet RPC encodings remain unchanged. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | -| `@bsv/wallet-toolbox` | `2.14.4` | `2.14.5` | patch | [API and usage](../packages/wallet/wallet-toolbox.md) | No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. | -| `@bsv/wallet-toolbox-client` | `2.14.4` | `2.14.5` | patch | [API and usage](../packages/wallet/wallet-toolbox-client.md) | No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. | -| `@bsv/wallet-toolbox-mobile` | `2.14.4` | `2.14.5` | patch | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. | -| `create-bsv-app` | `1.1.2` | `1.1.2` | none | [API and usage](../packages/helpers/create-bsv-app.md) | Existing CLI flags, network choices, and generated project structure are unchanged. Regenerate or update dependencies after the patched packages are published. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | +| Package | npm baseline | Source | Candidate | API | Migration | +| --------------------------------- | ------------ | -------- | --------- | --------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `@bsv/402-pay` | `0.3.2` | `0.3.3` | patch | [API and usage](../packages/middleware/402-pay.md) | No migration. Continue using the existing BRC-121 headers and replay store. Select SDK AuthFetch with auth/payment Express middleware for BRC-118; this package does not implicitly negotiate that protocol. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/air-gap` | `0.1.3` | `0.1.3` | none | [API and usage](../packages/helpers/air-gap.md) | No migration is required for valid BRC-141 v1 senders or decoders. Malformed, oversized, duplicate-heavy, or adversarial high-degree sessions now fail closed or are evicted within the documented limits instead of consuming unbounded resources. | +| `@bsv/amountinator` | `2.1.6` | `2.1.7` | patch | [API and usage](../packages/helpers/amountinator.md) | No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/auth` | `0.1.5` | `0.1.6` | patch | [API and usage](../packages/middleware/auth.md) | No valid proof bytes or public API shape changes. Client and server must use the same protocol tuple; security levels 0, 1, and 2 remain supported and select wallet consent policy while the explicit counterparty scopes derivation at every level. Proof and wallet adapters must supply plain own data fields and dense byte arrays; inherited, accessor-backed, sparse, or malformed runtime shapes fail closed. Structured bodies containing non-finite numbers or negative zero must be normalized to an unambiguous wire representation before signing. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/auth-express-middleware` | `2.2.8` | `2.3.0` | minor | [API and usage](../packages/middleware/auth-express-middleware.md) | Requires @bsv/sdk ^2.9.0 \|\| ^3.0.0. Set captureRawBody:true before body parsers when enabling multipart payment middleware. Default parsed-body integration remains available; nonempty non-multipart preimages remain compatible. Raw collection rejects compressed, oversized, slow and already-consumed bodies and enforces aggregate memory and pending-request bounds. Multipart boundary parameters must be signed exactly; receivers do not fall back to verification without the boundary. Configure public credential-free CORS preflight and expose the negotiation header. No deployment or publication is performed by this source candidate. For empty binary requests, upgrade SDK clients to the integrated 3.0.0 candidate and auth receivers to 2.3.0 together; older length-0 signatures/reconstructions need the paired correction. Nonempty preimages are unchanged. The retained SDK2 alternative starts at ^2.9.0; the integrated source graph now uses SDK3, including the paired BRC-118 correction. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/authsocket` | `2.1.8` | `2.1.9` | patch | [API and usage](../packages/messaging/authsocket.md) | No API or wire migration is required for valid JSON events. Existing numeric-key objects under byte-like names are unchanged; typed payment protocols recover historical byte objects at their explicit fields. Outbound non-JSON or ambiguous runtime values, including negative zero, nested undefined, sparse arrays, accessors, hidden/extra properties, and serialization hooks, now fail closed before signing. requestedCertificates is an SDK allowlist, not an application authorization verdict. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/authsocket-client` | `2.1.7` | `2.1.8` | patch | [API and usage](../packages/messaging/authsocket-client.md) | No API or event-wire migration. ESM/CommonJS consumers retain their existing SDK peer range. Browser distributors adopt the new UMD artifact after publication and must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. This source candidate does not publish or update current wallet release pins. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/btms` | `1.2.3` | `1.2.4` | patch | [API and usage](../packages/wallet/btms.md) | Existing local, mainnet, testnet, and number-array behavior is unchanged. TTN consumers select networkPreset teratestnet; all consumers should upgrade to @bsv/sdk 2.4.1 or later for byte-boundary compatibility. Valid canonical token amounts remain compatible. Audit historical tokens for signed, exponent, leading-zero, non-positive, or greater-than-9007199254740991 amount fields before rebuilding wallet state; public number-based operations now fail closed when an aggregate cannot be represented exactly. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/btms-permission-module` | `1.2.1` | `1.2.2` | patch | [API and usage](../packages/wallet/btms-permission-module.md) | Valid canonical BTMS flows remain compatible. Custom prompts must return literal true, authorization maps may contain at most 1,024 entries, and callers must provide dense signatures, safe exact totals, and canonical preimages; coercive or trailing-byte forms now fail closed. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/chirp` | `0.1.3` | `0.1.4` | patch | [API and usage](../packages/network/chirp.md) | No API, codec, object identifier, service schema, or wire-format migration. Use @bsv/chirp 0.1.3 with SDK 2.8.9 or later and Cloud Bucket 0.2.48 or Lite 0.1.45 for authenticated binary uploads, including full 4 MiB chunks. Custom authenticated transports must send the supplied owned byte body unchanged and supply ordinary HTTP framing; Content-Type remains signed through AuthFetch. Earlier service entrypoints do not parse staged binary bytes before authentication and require this service patch. Existing stored objects, sessions, advertisements and leases remain unchanged. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/did` | `0.2.6` | `0.3.0` | minor | [API and usage](../packages/helpers/did.md) | DID0.3 is a breaking pre-1.0 candidate: compressed identity-key input only; remote document/lifecycle options rejected. Use original signed BRC52 binary for envelopes, or explicitly verified compatible structured core. Wire authenticated BRC103/104 payload/control, durable atomic replay, consent, issuer/schema/purpose reliance and local status policy; callbacks alone are not full transport conformance. See docs/guides/identity-did-vc-migration.md. Proposed custom W3C mechanisms remain unregistered. Import BRC52 APIs/types from @bsv/did/brc52; identity-key DID and independent SD-JWT helpers stay at @bsv/did. | +| `@bsv/ecpm-permission-module` | `0.1.1` | `0.1.2` | patch | [API and usage](../packages/wallet/ecpm-permission-module.md) | Valid ecpm requests remain compatible. Authorization callbacks must return literal true; inherited, accessor-backed, malformed, or oversized inputs now fail closed. Hosts must keep grants below 1,024 entries and avoid more than 64 concurrently pending new authorization prompts per module instance. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/fund-wallet` | `1.5.2` | `1.5.3` | patch | [API and usage](../packages/helpers/fund-wallet.md) | No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/gasp` | `1.3.7` | `1.3.8` | patch | [API and usage](../packages/overlays/gasp.md) | No API or wire migration is required. In bidirectional mode, the receiver must already have the parent of a pushed child, request it during a subsequent synchronization round, or reject the graph because GASP v1 submitNode does not transmit the spentBy parent outpoint. Pull-only operation avoids that assumption. Custom storage and remote adapters must preserve exact request/node binding and bounded responses. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/lch` | `0.2.0` | `0.2.1` | patch | [API and usage](../packages/content/lch.md) | Replace buyer.quote(endpoint, request, issuer, keyGrants) with buyer.quote(verifiedSignedOffer, request, expectedSeller, keyGrants). JavaScript callers that still pass an endpoint string fail before transport I/O. Persist the returned plan and every partial settlement proof, retry with the same funded transaction, and configure a profile-aware agreementEvaluator before completion or recovery. Low-level unverified recovery results must not authorize key storage or content access. DNS endpoints require an address-pinning connector outside trusted browser environments. Distributors must retain THIRD_PARTY_NOTICES.md with the package; published BRC-170 remains authoritative if the implementation and standard differ. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/message-box-client` | `2.5.4` | `2.6.0` | minor | [API and usage](../packages/messaging/message-box-client.md) | Existing sender byte-array encoding and receiver byte forms remain compatible. Base64 must use the canonical standard alphabet and padding and decode to no more than 64 MiB. This receiver extension does not raise relay/intermediary limits or provide a durable refund/result journal; #548 and #503 retain those remaining milestones. Adopt the integrated SDK 3.0.0 candidate to obtain the independently tested BRC-29 recipient-key correction. No migration required for the new fields: the PeerMessage paymentOutcome and retained payment are optional, and messages without a payment are unchanged. To avoid losing payments, store any returned payment (validate it first; it is the raw envelope value) before acknowledging its message. Install @bsv/sdk 2.8.6 or later alongside this package; payments refused on earlier SDKs remain pending in the message box and can be accepted after upgrading. No BRC100 wire or wallet-data change. SDK consumers may retain ^2.8.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/overlay` | `2.6.2` | `2.6.3` | patch | [API and usage](../packages/overlays/overlay.md) | No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/overlay-discovery-services` | `2.2.6` | `2.2.7` | patch | [API and usage](../packages/overlays/overlay-discovery-services.md) | No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/overlay-express` | `2.7.3` | `2.7.4` | patch | [API and usage](../packages/overlays/overlay-express.md) | No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/overlay-topics` | `1.9.1` | `2.0.0` | major | [API and usage](../packages/overlays/overlay-topics.md) | Topics2 is a breaking retirement candidate. Remove DIDTopicManager/createDIDLookupService/DIDRecord/DIDQuery and tm_did/ls_did usage. Resolve immutable identity-key DIDs offline; explicitly configured tm_identity/ls_identity supplies certificate discovery, not DID-document mutation or automatic trust. Preserve existing database/history and plan service rollout without automatic replacement installation. See docs/guides/identity-did-vc-migration.md. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. Mandala is a clean break with no data migration: the old MandalaToken/MandalaAdmin wire format is no longer admitted and 1.x Mandala rows are not read or converted, so start Mandala on a new database with new deploys (existing on-chain outputs are not spent or deleted). MandalaTopicManager drops adminWallet, adminProtocolID and stateStore.isAdminOutpoint and requires trustedIssuers (non-empty compressed lowercase public keys) and engineOutputs, with optional membership, membershipExempt and onOwnerRepair; pass the same MandalaStorageManager (a MandalaStateStore) to admission and lookup. Refusals are MandalaReject { code, reason } from the Reasons catalog; do not match on old error text. MandalaLinkagePayload is replaced by MandalaEnvelope with encodeEnvelope/decodeEnvelope; the register action is replaced by a deploy at output 0 with a deploySig. foldAction, defaultAssetState, AssetAdminState, FoldContext and MandalaTokenRecord use tokenId (_0) instead of assetId, drop issuerIdentityKey and add feeRatePerKb. Persisted schema (section 6.6): new mandalaOwners and mandalaAuthorities collections; mandalaTokens, mandalaMetadata, mandalaAssetStates and mandalaAdminHistory are keyed by tokenId, metadata holds the decoded deploy payload, and history rows store kind, detailsHex, commitment and delta (plus optional frozenAmount/frozenOwner on freezes). De-trusting an issuer key revokes the authority coins it holds; rotate keys by moving authority coins first. Operators register tm_mandala_registry/ls_mandala_registry explicitly. The infra/overlay-server wiring follows after publication (P1b). | +| `@bsv/paymail` | `2.4.9` | `2.4.10` | patch | [API and usage](../packages/messaging/paymail.md) | None. ESM consumers are unaffected. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/payment-express-middleware` | `2.1.8` | `2.2.0` | minor | [API and usage](../packages/middleware/payment-express-middleware.md) | Requires @bsv/sdk ^2.9.0 \|\| ^3.0.0. Set enableMultipart:true behind @bsv/auth-express-middleware captureRawBody:true installed before parsers. Support is advertised only on verified raw-byte requests. Configure maxPaymentBytes and maxPaymentBodyBytes with matching proxy/auth limits, and share a durable replay store across processes. Preserve recovery context after any submitted payment: the released ERR_PAYMENT_FAILED 400 does not prove rollback, and replay-store 503 is also ambiguous. Header-only deployments remain supported. See the BRC-118 guide. The retained SDK2 alternative starts at ^2.9.0; the integrated source graph now uses SDK3, including the paired BRC-118 correction. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/sdk` | `2.8.10` | `3.0.0` | major | [API and usage](../packages/sdk/bsv-sdk.md) | Use a BRC-100 wallet that honors noSend and sendWith and implements abortAction for automatic payments. Deploy raw-byte auth and multipart payment receivers first. Header-only or GET/HEAD requests whose payments require multipart fail before broadcast; select an application-supported body-bearing route explicitly. Serialize file-bearing FormData to owned bytes and retain its exact Content-Type; text-only FormData retains URL encoding. Reconcile submitted/uncertain payments before creating another spend, including after changed requirements, cancellation or proxy 413/431. Existing small header payments and nonempty non-multipart signing retain their wire format. Empty byte bodies use the BRC-104 -1 sentinel; clients and auth receivers that signed/reconstructed length 0 must adopt SDK 2.9.0 and auth middleware 2.3.0 together for those requests. BRC-29 receivers now derive their own child key. See the BRC-118 guide for limits and known external-peer differences. This candidate is not published and does not change current wallet-release pins. The included 2.8.3 discovery/browser-fetch and signed action-history repairs preserve the existing BRC100 API, historical wire bytes and account data. Older compatible clients retain their calls; applications bundling the affected 2.8.x binary client need the signed-history repair as well as the wallet. No ecosystem-wide migration is required for those compatibility fixes. SDK 2.8.3 was published by the protected SDK-only release workflow with verified registry integrity and provenance. The included 2.8.11 certificate-issuance compatibility repair requires no API, wire or wallet-data migration. SDK3 is a separate breaking retirement candidate: remove identity/DIDTokenValidation imports and use @bsv/did BRC202 identity-key resolution plus optional @bsv/did/brc52 original-signature envelopes. Retain historical data and recover authenticated identity bindings; do not relabel serial DIDs. Existing acquisition-result validation fixes remain. First-party SDK3 peer alternatives require complete packed qualification and owner coordination; preserved SDK2 ranges are not silently narrowed. See docs/guides/identity-did-vc-migration.md. | +| `@bsv/simple` | `0.6.0` | `0.7.0` | minor | [API and usage](../packages/helpers/simple.md) | Simple0.7 is a breaking pre-1.0 candidate. Remove mutable DID/resolver/provider configuration, proxy handlers and DIDError/types; use DID.fromIdentityKey/resolve and wallet.getDID. CredentialIssuer.issue returns {credential,keyringForSubject}; verify accepts exact JSON or UTF8 bytes and returns structured cryptographic results. The HTTP verify route now requires {credential: originalEnvelopeJson} with original text as a string; parsed objects return HTTP400. Web request decoding rejects duplicate members and invalid UTF8, and custom parsed-body adapters must establish equivalent strict decoding. Preserve received text and inspect verification.verified, not HTTP success. Local revocation record status is retained/unknown, not inferred chain revocation. See docs/guides/identity-did-vc-migration.md. Replace createServerWalletHandler() deployments with createServerWalletHandler({ authorize: async ({ action, headers }) => authenticatedSessionCanUseAction(headers, action) }). The callback must return literal true for each status, create, request, receive, balance, outputs, or reset action; omission now returns HTTP 403 for every action. Roll out the authentication layer and callback with the package, update anonymous probes or automation, and apply the same policy to every replica. Do not emulate the old public behavior with an unconditional authorize: () => true callback. Valid recipient derivations and authenticated Message Box peers remain supported; malformed, wrong-owner, or transaction-mutated flows now fail closed. New CredentialSchema and Certifier records use canonical 32-byte types. Current SDK wallet methods reject historical short types, so do not put migration aliases in wallet list, acquire, prove, or relinquish calls. Export affected records through the storage version that created them, authenticate them offline against the exact locally configured identifier, and reissue/import canonical replacements; no legacy certificate is rewritten automatically. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/. Persisted local field ordering requires no migration; transport adapters preserve original certificate signing bytes. | +| `@bsv/templates` | `1.10.2` | `2.0.0` | major | [API and usage](../packages/helpers/templates.md) | Breaking. Removed exports: MandalaToken, MandalaTokenDecoded, MandalaAdmin, ADMIN_PROTOCOL, MandalaAdminDecoded, MandalaActionDetails, MandalaActionKind, MandalaAdminLockParams, MandalaAdminUnlockParams and AssetMetadata. Removed subpath @bsv/templates/mandala-signing.ts. The @bsv/templates/mandala-encoding.ts subpath keeps only createMinimallyEncodedScriptChunk, decodeScriptNum and decodeScriptNumChunk; encodeScriptNum, encodeAssetId and decodeAssetId are removed. Replace MandalaToken/MandalaAdmin with Bsv21Binary: deploy, authority and value outputs are lock(tokenId \| null, amount, pubKeyHash, payload?) with bigint amounts and _0 token ids (tokenIdFromString/tokenIdToString), decoded with Bsv21Binary.decode. Mandala admin actions are committed by sha256 of strict-CBOR details in the authority output payload, not by a commitment-derived key. This is a clean break with no data migration: tokens and admin chains in the old format are not readable by 2.0.0. @bsv/overlay-topics 2.0.0 consumes this format; the infra/overlay-server wiring follows after publication. SDK consumers may retain ^2.1.6 or install the SDK3 candidate; SDK3 removes the obsolete DID token API (see docs/guides/identity-did-vc-migration.md). | +| `@bsv/teranode-listener` | `1.1.6` | `1.1.7` | patch | [API and usage](../packages/network/teranode-listener.md) | No API migration is required for valid consumers: raw callbacks remain the default and decoding is opt-in with decodeMessages: true. Configuration arrays and callbacks are snapshotted at construction, boolean controls must be literal booleans, and malformed or duplicate topics, addresses, keys, and unsupported properties now fail closed. usePrivateDHT: false now actually omits the DHT service. The published mainnet PNET value is transport compatibility data, not a publisher credential; decoded sender and payload fields remain untrusted and security-critical claims require independent validation. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/verifast` | `0.3.6` | `0.3.7` | patch | [API and usage](../packages/sdk/verifast.md) | Valid typed verification calls and worker protocols remain compatible. Custom module factories and WASM adapters must return the exact documented binary and boolean shapes; coercive network, height, flag, byte, batch, verdict, lifecycle, or disposed-instance values now fail closed. Keep THIRD_PARTY_NOTICES.md and LICENSES/ with every JavaScript and WebAssembly distribution. SDK consumers may retain ^2.1.8 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/wallet-helper` | `0.1.8` | `0.1.9` | patch | [API and usage](../packages/helpers/wallet-helper.md) | No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/wallet-relay` | `0.5.1` | `0.5.2` | patch | [API and usage](../packages/wallet/wallet-relay.md) | Enable signed QR codes and distribute only HTTPS pairing origins and relay API URLs. Root-relative API paths and loopback HTTP remain supported. Configure onApprovalRequired for every method not deliberately listed in autoApproveMethods; unsigned pairing URIs and implicit approval are no longer accepted. Existing relay sessions, custom RPC method names, and supported wallet RPC byte encodings remain valid, and host applications continue to provide their matching Express runtime and type graph. Version 0.5 defaults to two missed pongs; set maxMissedHeartbeats to 1 to retain the prior heartbeat policy. Heartbeat intervals must fit the Node timer range. Preserve Cache-Control: no-store at proxies, prefer the stable bsv-wallet-relay plus token WebSocket subprotocols over the legacy token query parameter, and configure edge rate limiting in addition to the bounded in-process defaults. The 24-hour connected-session lifetime no longer renews on reconnect, malformed wallet calls fail with code 400, and WalletRelayClient retains at most 100 log entries unless maxLogEntries is configured. The QR remains a short-lived bearer invitation: keep it private and require explicit operation approval. Relay envelopes, cryptographic framing, and wallet RPC encodings remain unchanged. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | +| `@bsv/wallet-toolbox` | `2.14.4` | `2.15.0` | minor | [API and usage](../packages/wallet/wallet-toolbox.md) | Unpublished journal generation helpers now require explicit receipt capacity/lifetime policy and refuse mismatched or earlier unregistered layouts. No registered migration or reader capability is added; capture publication, continuity-floor collection and receiver integration remain unfinished. Internal journal bootstrap callers must supply a fixed row allowance; this unpublished auxiliary layout rejects earlier unregistered generations rather than silently resetting their counters. Registered migration, reset and full resource-retention policy remain unfinished. The internal journal foundation adds no registered migration or public reader advertisement; full quota/capture/floor/receiver integration and full544 acceptance remain incomplete. The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. Use syncFromReaderResumable for the new local atomic-page path; legacy/custom or remote destinations retain exclusive fallback. Cancellation drains in-flight work and resumes from the durable destination checkpoint. Custom providers opt into concurrent reads/atomic pages only when those contracts hold; safe proof persistence additionally requires compareAndSetProvenTxProof. Configured proof services must validate active 80-byte headers. The original legacy page/proof subset needs no migration. The legacy timestamp/offset path may reread equal-time rows and does not provide a coherent source snapshot or streaming archive. Choose maxRoughSize deliberately for device/proxy memory budgets. Current wallet releases remain on their published dependency pins. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Retained views occupy a pool connection and require awaited close/closed cleanup; configure driver deadlines separately. IndexedDB retention remains unsupported. The retained-view primitive introduces no schema or wire migration. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. Apply 2026-09-30-002 add snapshot archive staging through the normal SQL migrate() entry point. Its capacity, manifest and page tables are auxiliary and excluded from BRC-38; no standard-table or legacy-checkpoint change is made. Current staging ceilings are eight handles, 128 MiB reserved globally, 32 MiB per capture, 1 MiB per page, 4096 pages and one hour lifetime. Reservations remain occupied through physical page cleanup. These preliminary limits do not qualify larger-wallet remote export; canonical portable semantic validation and operator resource policy are still required. The local capture controller requires a dedicated reader and an independent staging pool, with schema migrations outside active capture windows; it exposes no RPC or canonical BRC-38 output. Existing sync/backup behavior and browser/mobile adapters are unchanged by this staging component. The internal receipt directory requires no additional schema migration and exposes no RPC capability. Its one-MiB metadata budget does not replace a transport envelope limit; inclusion integrity is separate from portable semantic/proof validation. The existing unpublished 2.15 minor candidate covers these packed core bytes; no package is published by this change. Apply the additive 2026-09-30-003 add snapshot archive requests migration through migrate(); its auxiliary receipts stay outside portable wallet data. Pending requests use the existing archive capacity counters. Close outstanding requests before removing the request schema. The internal controller caps source lifetime at five minutes or the request remaining deadline and keeps the archive deadline fixed. Its per-process reader bound is distinct from shared logical staging quotas; driver cleanup deadlines remain an operator concern. The authenticated archive transport below consumes these internal components; remote row/manager integration and contention performance qualification remain incomplete. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. Apply 2026-10-01-001 add snapshot archive source owners before service capture. Stop and drain captures before downgrading; the migration refuses removal while owners remain and older draft binaries must not capture concurrently. Old unguarded owners retain ownership after unproved process loss. Standard tables and BRC-38/39 bytes are unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Apply 2026-10-01-002 add snapshot archive source guards through migrate(). Preserve the eight persistent slot bindings and SQLite sidecar guard files; never replace a bound file to free quota. New captures require better-sqlite3 WAL or the same actual MySQL server/database. Missing or changed backend identity refuses recovery. Drain all captures before downgrade; old unguarded owners require explicit source cleanup. Distributed filesystems, database relocation/restoration, PXC/load-balanced failover and global physical-connection limits require separate qualification. Reader advertisement stays disabled pending complete lifecycle qualification. After a source-cleanup error, do not reuse the affected provider for snapshot capture; retain its owner reservation until backend recovery proves physical closure. This correction adds no schema or wire change. Apply 2026-10-01-003 add snapshot profile key indexes through migrate(). This explicit nontransactional migration commits bounded 256-row auxiliary key/progress batches and journals only completion; preserve partial objects for retry, verify a terminated migrator before recovering any stale Knex lock, and drain readers before down. Standard rows/indexes and BRC-38/cursor bytes remain unchanged. Relationship/global-table indexing, commit-order high-water and nonblocking IndexedDB remain incomplete. Apply 2026-10-01-004 add snapshot relation key indexes through migrate(). Install/removal observers and producers preserve both ownership bases, including inconsistent mappings for closure refusal. Preserve partial auxiliary DDL/progress for retry, exclude other migrators before lock recovery, and drain readers before down. This migration remains outside BRC-38. Certificate-field/proof indexing, committed-change ordering, streaming and staged import remain incomplete; reader advertisement remains disabled. Apply 2026-10-01-005 add snapshot certificate field key indexes through migrate(). MySQL source and auxiliary tables require InnoDB and matching varchar(100) character set/collation; SQLite verifies the existing complete unique key and built-in BINARY/NOCASE/RTRIM order. Preserve partial owned DDL and committed bootstrap positions, exclude other migrators before verified lock recovery, and drain retained readers before down. Standard tables, indexes, legacy OFFSET and BRC-38/cursor bytes remain unchanged; proof/request indexing and the full program remain incomplete. Apply 2026-10-01-006 add snapshot global reference indexes through migrate(). Its reference edges, exact counts, presence guards and progress are auxiliary and excluded from BRC-38. Preserve partial DDL/committed positions; prove a stopped migrator and exclude others before lock recovery. MySQL requires standard unsigned/text metadata and InnoDB with explicit RESTRICT/NO ACTION mutations; implicit CASCADE/SET NULL actions are refused before DDL. Drain readers before down. The complete journal and progress are adopted within each pinned ordinary/archive view; source rows/indexes, legacy OFFSET and wire bytes stay unchanged. Commit ordering, tombstones, nonblocking IDB, remote destinations, streaming/staged portability and full acceptance remain incomplete; reader advertisement remains off. Apply 2026-10-02-001 repair snapshot SQLite conflict maintenance through migrate(). Preserve partial owned generation/progress for recovery and prove the stopped migrator before stale lock recovery. New readers use source queries while pending and adopt v2 inside the same retained view only after journal publication. Do not downgrade older snapshot binaries against invalidated legacy progress. Ordinary down refuses without deleting source rows; use a forward migration. Explicit dropAllData still deletes the complete wallet, including unpublished generation state. Freed SQLite pages may remain allocated for reuse; bounded retirement does not promise file shrink or full resource-budget acceptance. The SQLite migration-journal lookup correction needs no schema or wire migration; retain the existing tableName/schemaName binding. Postgres records these auxiliary snapshot migrations as no-ops and advertises no retained or paged source. Future Postgres snapshot support needs new forward migrations rather than rewriting the recorded entries. Internal openSnapshotJournalSource requires an existing file-backed better-sqlite3 WAL database or static mysql2 connection and an already complete explicitly owned journal generation. It registers no migration or incremental capability. A killed read view must be captured anew; retained prefix receipts do not reopen it. Runtime quotas, registered journal lifecycle, delta paging and receiver integration remain pending. Internal maintainSnapshotJournal owns generation validation and configured Knex migration exclusion before each bounded floor/collection transaction. Raw operator DDL remains outside that exclusion contract. Cancellation keeps shared capture admission until rollback or committed-result recovery and physical pool cleanup; unproved cleanup fences retained admission and remains observable through destruction. Runtime quotas, registered journal lifecycle/recovery and incremental reader advertisement remain unfinished; no new registered migration or public wire format is introduced. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation. | +| `@bsv/wallet-toolbox-client` | `2.14.4` | `2.15.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-client.md) | The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. Use the additive syncFromReaderResumable API and its progress/cancellation contracts; custom providers keep the exclusive fallback until they implement the capability contract. No IndexedDB schema migration is required. Source timestamps remain inclusive rather than a coherent snapshot; equal-time boundary rows can be reread. Select explicit page bytes for constrained devices. Publication and downstream wallet adoption are separate actions. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Do not assume retained-view support from the types; inspect the concrete local provider capability. Existing scoped capture remains unchanged. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Backend-bound orphan recovery remains required before reader advertisement. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation. | +| `@bsv/wallet-toolbox-mobile` | `2.14.4` | `2.15.0` | minor | [API and usage](../packages/wallet/wallet-toolbox-mobile.md) | The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. The additive syncFromReaderResumable API uses atomic local pages only for capable local providers; remote/custom destinations keep the serialized fallback. Existing mobile runtime/crypto backend registration and RPC wires remain unchanged. The original legacy page subset needs no migration. Reconcile cancellation and lost acknowledgements using the durable checkpoint; do not infer coherent source snapshots. Current wallet releases are not required to consume this candidate. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Do not assume retained-view support from the types; inspect the concrete local provider capability. Existing scoped capture remains unchanged. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Backend-bound orphan recovery remains required before reader advertisement. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation. | +| `create-bsv-app` | `1.1.2` | `1.1.2` | none | [API and usage](../packages/helpers/create-bsv-app.md) | Existing CLI flags, network choices, and generated project structure are unchanged. Regenerate or update dependencies after the patched packages are published. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. | `none` means the source manifest matches the recorded npm baseline. Any other value is an unpublished candidate. Publication, tags, releases, registry @@ -70,8 +70,8 @@ explicitly authorized operations. - Package documentation: [docs/packages/middleware/402-pay.md](../packages/middleware/402-pay.md) - Source: [packages/middleware/402-pay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/402-pay) -- Release note: Adds an exact-tarball Vite and esbuild contract for the browser-safe client entry point, including a bundle-size ratchet and an assertion that server exports never leak into browser consumers. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Standardizes first-party author metadata on the current BSV Association name. Binds BRC-121 pricing and internalization to the transaction declared by the BRC-95 Atomic BEEF subject, removes unrelated included branches, rejects trailing bytes and non-atomic envelopes, and requires affirmative wallet acceptance before serving paid content. Adds independent bounded atomic transaction replay claims so conforming BRC-100 wallets that omit the non-public isMerge detail cannot authorize duplicate access, preserves actual overpayments in middleware receipts, uses fixed bounded wallet descriptions, makes diagnostics opt-in, and returns an unchallenged 503 after ambiguous wallet or replay-store failures to avoid inducing a second spend. Aligns the development-only Vitest runner and V8 coverage provider at 4.1.11. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: Custom clients must send a strictly framed BRC-95 Atomic BEEF envelope whose subject is the payment transaction; plain BEEF is no longer accepted. Legacy Atomic BEEF containing unrelated branches remains compatible because the server reduces it to the declared subject and dependency closure. The default replay store is bounded and process-local; production services with more than one serving process or node must inject the same durable atomic PaymentReplayStore everywhere. Low-level validators should retain one wallet object or pass an explicit store. Applications that relied on implicit console diagnostics must supply the optional structured logger. Treat an unchallenged 503 after payment submission as ambiguous and reconcile the transaction before requesting another payment. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Documents the protocol boundary between this package's BRC-121 header payments and the separate BRC-105/BRC-118 authenticated multipart integration. Runtime and wire behavior are unchanged. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: No migration. Continue using the existing BRC-121 headers and replay store. Select SDK AuthFetch with auth/payment Express middleware for BRC-118; this package does not implicitly negotiate that protocol. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ------------------------------------------ | ---------------------------------------------- | @@ -94,8 +94,8 @@ explicitly authorized operations. - Package documentation: [docs/packages/helpers/amountinator.md](../packages/helpers/amountinator.md) - Source: [packages/helpers/amountinator](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/helpers/amountinator) -- Release note: Validates finite monetary values, normalized nonempty currency codes, integer decimal precision, grouping flags, converter rates, and conversion results, and formats negative sub-unit amounts from their absolute magnitude without losing the sign. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: Valid finite inputs retain the public API. Currency identifiers are trimmed and normalized to uppercase; callers that passed non-finite values, empty currencies, coercive options, invalid decimal precision, or non-finite converter results must normalize or reject them before calling because those values now fail closed. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------- | -------------------------------------------- | @@ -116,8 +116,8 @@ explicitly authorized operations. - Package documentation: [docs/packages/middleware/auth-express-middleware.md](../packages/middleware/auth-express-middleware.md) - Source: [packages/middleware/auth-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware) -- Release note: Removes middleware header byte/count ceilings and excludes received HTTP headers from authenticated request-body budgets, preserving signed header bytes. HTTP server and edge layers own transport capacity. Canonical framing, authentication, invalid-header rejection, body budgets and response signing remain. Requires SDK 2.8.5 and delegates its Peer general-payload capacity to the HTTP layer, avoiding an indirect SDK envelope ceiling for received headers. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: No BRC100 call, wire or wallet-data migration. Configure and validate transport header limits at the HTTP server, CDN, proxy or WAF; maxRequestBytes no longer caps received headers. Clients needing larger signed responses require corresponding client capacity, including SDK 2.8.5. Published 2.2.8 is verified against protected release 36052862859 and its immutable source artifacts. Upgrade the required SDK peer to 2.8.5; custom ExpressTransport/Peer setups should pass maxGeneralPayloadBytes=null when the HTTP layer owns payload capacity. SDK consumers may retain ^2.8.5 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Adds opt-in bounded raw request collection for BRC-118, authenticates the whole body and exact multipart Content-Type before extraction, restores decoded non-multipart application payloads after authentication, and preserves Express set(object) response-header behavior. Normalizes zero-length dense byte arrays and Uint8Array/Buffer bodies to the BRC-104 -1 sentinel, matching AuthFetch when express.raw runs before authentication. Frames typed-array bodies from one copied byte vector, so length metadata and serialized payload cannot disagree. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: Requires @bsv/sdk ^2.9.0 || ^3.0.0. Set captureRawBody:true before body parsers when enabling multipart payment middleware. Default parsed-body integration remains available; nonempty non-multipart preimages remain compatible. Raw collection rejects compressed, oversized, slow and already-consumed bodies and enforces aggregate memory and pending-request bounds. Multipart boundary parameters must be signed exactly; receivers do not fall back to verification without the boundary. Configure public credential-free CORS preflight and expose the negotiation header. No deployment or publication is performed by this source candidate. For empty binary requests, upgrade SDK clients to the integrated 3.0.0 candidate and auth receivers to 2.3.0 together; older length-0 signatures/reconstructions need the paired correction. Nonempty preimages are unchanged. The retained SDK2 alternative starts at ^2.9.0; the integrated source graph now uses SDK3, including the paired BRC-118 correction. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ------------------------------------ | ---------------------------------------- | @@ -140,8 +140,8 @@ explicitly authorized operations. - Package documentation: [docs/packages/messaging/authsocket-client.md](../packages/messaging/authsocket-client.md) - Source: [packages/messaging/authsocket-client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/authsocket-client) -- Release note: Contains authentication and application callback failures, caps authentication concurrency, snapshots strict JSON before signing, serializes real typed arrays portably, preserves supported signed event JSON exactly, and ships the complete SDK incorporated-material notice archive with a retained UMD notice banner. Standardizes first-party author metadata on the current BSV Association name. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: No API or wire migration is required for valid JSON events. Existing numeric-key objects under byte-like names are unchanged; typed payment protocols recover historical byte objects at their explicit fields. Outbound non-JSON or ambiguous runtime values, including negative zero, nested undefined, sparse arrays, accessors, hidden/extra properties, and serialization hooks, now fail closed before signing. The connect event/connected property report Socket.IO transport state, not completed BRC-103 authentication; wait for verified application traffic when local behavior requires a known unpinned server. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Refreshes the distributed UMD browser bundle with the integrated SDK implementation. The rebuilt bundle differs from published 2.1.7; the patch candidate keeps those new bytes under a new version. AuthSocket source APIs and event serialization are unchanged. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: No API or event-wire migration. ESM/CommonJS consumers retain their existing SDK peer range. Browser distributors adopt the new UMD artifact after publication and must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. This source candidate does not publish or update current wallet release pins. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ----------------------------------- | --------------------------------------- | @@ -213,8 +213,8 @@ CLI entry points: `{"chirp":"./dist/cli.js"}`. - Package documentation: [docs/packages/helpers/fund-wallet.md](../packages/helpers/fund-wallet.md) - Source: [packages/helpers/fund-wallet](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/helpers/fund-wallet) -- Release note: Requires the funding wallet's internalizeAction result to contain accepted: true before reporting a funded transaction as successfully internalized. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: No public API migration is required for conforming wallets. Custom WalletInterface adapters must return the literal accepted: true verdict after successful internalization; refusals and malformed or coercive results now throw instead of being reported as success. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. CLI entry points: `{"fund-metanet":"./dist/index.mjs"}`. @@ -252,8 +252,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/messaging/message-box-client.md](../packages/messaging/message-box-client.md) - Source: [packages/messaging/message-box-client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/message-box-client) -- Release note: listMessages() and listMessagesLite() report paymentOutcome ('internalized', 'failed', 'skipped' or 'no-wallet-outputs') on each message that carried a payment, and keep that payment on the returned message unless the wallet accepted it (issue #503). Previously the payment envelope was always removed and a failed internalization was only logged, so acknowledging a listed message could delete the only copy of a payment. Both PeerMessage fields are optional and absent on messages without a payment. This release also carries the @bsv/sdk peer-floor raise from ^2.8.0 to ^2.8.6: SDK 2.8.0 through 2.8.5 derive the recipient key in Brc29RemittanceModule.acceptSettlement without forSelf, so PeerPayClient.acceptPayment() and rejectPayment() fail with brc29.internalize_failed for every valid incoming payment, and SDK 2.8.6 fixes acceptance. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: No migration required for the new fields: the PeerMessage paymentOutcome and retained payment are optional, and messages without a payment are unchanged. To avoid losing payments, store any returned payment (validate it first; it is the raw envelope value) before acknowledging its message. Install @bsv/sdk 2.8.6 or later alongside this package; payments refused on earlier SDKs remain pending in the message box and can be accepted after upgrading. No BRC100 wire or wallet-data change. SDK consumers may retain ^2.8.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Accepts canonical, bounded base64 transaction strings at PeerPay message receive boundaries alongside portable number arrays and legacy numeric-key JSON byte objects. Applies one validation path to list, live and indexed acceptance and rejects malformed or oversized encodings before wallet work or acknowledgement. listMessages() and listMessagesLite() report paymentOutcome ('internalized', 'failed', 'skipped' or 'no-wallet-outputs') on each message that carried a payment, and keep that payment on the returned message unless the wallet accepted it (issue #503). Previously the payment envelope was always removed and a failed internalization was only logged, so acknowledging a listed message could delete the only copy of a payment. Both PeerMessage fields are optional and absent on messages without a payment. This release also carries the @bsv/sdk peer-floor raise from ^2.8.0 to ^2.8.6: SDK 2.8.0 through 2.8.5 derive the recipient key in Brc29RemittanceModule.acceptSettlement without forSelf, so PeerPayClient.acceptPayment() and rejectPayment() fail with brc29.internalize_failed for every valid incoming payment, and SDK 2.8.6 fixes acceptance. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: Existing sender byte-array encoding and receiver byte forms remain compatible. Base64 must use the canonical standard alphabet and padding and decode to no more than 64 MiB. This receiver extension does not raise relay/intermediary limits or provide a durable refund/result journal; #548 and #503 retain those remaining milestones. Adopt the integrated SDK 3.0.0 candidate to obtain the independently tested BRC-29 recipient-key correction. No migration required for the new fields: the PeerMessage paymentOutcome and retained payment are optional, and messages without a payment are unchanged. To avoid losing payments, store any returned payment (validate it first; it is the raw envelope value) before acknowledging its message. Install @bsv/sdk 2.8.6 or later alongside this package; payments refused on earlier SDKs remain pending in the message box and can be accepted after upgrading. No BRC100 wire or wallet-data change. SDK consumers may retain ^2.8.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ----------------------------------- | --------------------------------------- | @@ -279,8 +279,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-discovery-services.md](../packages/overlays/overlay-discovery-services.md) - Source: [packages/overlays/overlay-discovery-services](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-discovery-services) -- Release note: Adds the unique MongoDB _id tie breaker to discovery pagination so bounded overflow probes preserve stable ordering. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -291,8 +291,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/overlays/overlay-express.md](../packages/overlays/overlay-express.md) - Source: [packages/overlays/overlay-express](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/overlay-express) -- Release note: Pages built-in discovery overflow probes within the discovery service 1000-row query bound, retaining the configured engine ceiling and explicit overflow failure. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------------- | ---------------------------------------------------- | @@ -334,8 +334,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/middleware/payment-express-middleware.md](../packages/middleware/payment-express-middleware.md) - Source: [packages/middleware/payment-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/payment-express-middleware) -- Release note: Removes middleware payment-header size rejection so received payments are validated regardless of header size. Retains the deprecated maxPaymentHeaderBytes option as an ignored source-compatibility field. Atomic BEEF framing, canonical base64, derivation verification, pricing, wallet acceptance and atomic replay protection remain. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: No BRC100 call, wire or wallet-data migration. maxPaymentHeaderBytes is now ignored, including existing explicit values: move transport policy to the HTTP server, CDN, proxy or WAF and remove the deprecated option. Validate the complete HTTP route for supported payment proofs. Published 2.1.8 is verified against protected release 36052862859 and its immutable source artifacts. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Adds receiver-first BRC-118 negotiation and bounded authenticated multipart payment extraction. Restores the original application payload and media type while retaining the existing BRC-105 amount, derivation, Atomic BEEF, internalization and replay checks. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: Requires @bsv/sdk ^2.9.0 || ^3.0.0. Set enableMultipart:true behind @bsv/auth-express-middleware captureRawBody:true installed before parsers. Support is advertised only on verified raw-byte requests. Configure maxPaymentBytes and maxPaymentBodyBytes with matching proxy/auth limits, and share a durable replay store across processes. Preserve recovery context after any submitted payment: the released ERR_PAYMENT_FAILED 400 does not prove rollback, and replay-store 503 is also ambiguous. Header-only deployments remain supported. See the BRC-118 guide. The retained SDK2 alternative starts at ^2.9.0; the integrated source graph now uses SDK3, including the paired BRC-118 correction. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ------------------------------------ | ---------------------------------------- | @@ -346,8 +346,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/sdk/bsv-sdk.md](../packages/sdk/bsv-sdk.md) - Source: [packages/sdk](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/sdk) -- Release note: WalletClient and the binary BRC-100 wire no longer reject every acquireCertificate issuance: result validation required the returned fields to equal the request's, but issuance sends plaintext values and the certificate comes back with each value encrypted. An issuance result is now bound to exactly the requested field names; a direct acquisition is still bound to its requested encrypted values exactly. Removes the obsolete serial-DID token validation module and deep import. Native identity/certificate, wallet, transaction and authentication behavior is retained. Repairs cold BasePoint/JacobianPoint leaf exports with original-class initialization wrappers and exact declarations. The ESM UMD facade retains the established global effect and canonical class identities; the standalone classic bundle is preserved. -- Migration: SDK3 is a separate breaking retirement candidate: remove identity/DIDTokenValidation imports and use @bsv/did BRC202 identity-key resolution plus optional @bsv/did/brc52 original-signature envelopes. Retain historical data and recover authenticated identity bindings; do not relabel serial DIDs. Existing acquisition-result validation fixes remain. First-party SDK3 peer alternatives require complete packed qualification and owner coordination; preserved SDK2 ranges are not silently narrowed. See docs/guides/identity-did-vc-migration.md. +- Release note: Adds BRC-118 negotiated, byte-preserving multipart payments to AuthFetch with bounded header/body budgets, prepare-before-broadcast validation, one-transaction submission/retry, cancellation and typed permanent recovery outcomes. Preserves nonempty non-multipart signature preimages and fixes BRC-29 recipient child-key derivation. Adds independent Python wire/preimage vectors and composed authenticated HTTP/proxy regression coverage. Preserves the configured BRC100 caller originator during automatic wallet discovery, so the HTTP WalletWire probe satisfies the existing transport caller binding. Binds the default HTTP JSON fetch receiver for browsers while retaining custom HTTP clients. Keeps transport preference, bounded discovery, operation timeout behavior, validation and origin checks unchanged. Restores signed listActions net amounts using their historical int64 wire encoding, matching existing JSON validation, while retaining canonical unsigned counts/lengths and nonnegative individual output values. Preserves native fetch body serialization and inferred Content-Type on ordinary HTTP fallback, including cached peers. WalletClient and the binary BRC-100 wire no longer reject every acquireCertificate issuance: result validation required the returned fields to equal the request's, but issuance sends plaintext values and the certificate comes back with each value encrypted. An issuance result is now bound to exactly the requested field names; a direct acquisition is still bound to its requested encrypted values exactly. Removes the obsolete serial-DID token validation module and deep import. Native identity/certificate, wallet and transaction APIs remain; the BRC-118 authentication changes above are included in the major candidate. Repairs cold BasePoint/JacobianPoint leaf exports with original-class initialization wrappers and exact declarations. The ESM UMD facade retains the established global effect and canonical class identities; the standalone classic bundle is preserved. +- Migration: Use a BRC-100 wallet that honors noSend and sendWith and implements abortAction for automatic payments. Deploy raw-byte auth and multipart payment receivers first. Header-only or GET/HEAD requests whose payments require multipart fail before broadcast; select an application-supported body-bearing route explicitly. Serialize file-bearing FormData to owned bytes and retain its exact Content-Type; text-only FormData retains URL encoding. Reconcile submitted/uncertain payments before creating another spend, including after changed requirements, cancellation or proxy 413/431. Existing small header payments and nonempty non-multipart signing retain their wire format. Empty byte bodies use the BRC-104 -1 sentinel; clients and auth receivers that signed/reconstructed length 0 must adopt SDK 2.9.0 and auth middleware 2.3.0 together for those requests. BRC-29 receivers now derive their own child key. See the BRC-118 guide for limits and known external-peer differences. This candidate is not published and does not change current wallet-release pins. The included 2.8.3 discovery/browser-fetch and signed action-history repairs preserve the existing BRC100 API, historical wire bytes and account data. Older compatible clients retain their calls; applications bundling the affected 2.8.x binary client need the signed-history repair as well as the wallet. No ecosystem-wide migration is required for those compatibility fixes. SDK 2.8.3 was published by the protected SDK-only release workflow with verified registry integrity and provenance. The included 2.8.11 certificate-issuance compatibility repair requires no API, wire or wallet-data migration. SDK3 is a separate breaking retirement candidate: remove identity/DIDTokenValidation imports and use @bsv/did BRC202 identity-key resolution plus optional @bsv/did/brc52 original-signature envelopes. Retain historical data and recover authenticated identity bindings; do not relabel serial DIDs. Existing acquisition-result validation fixes remain. First-party SDK3 peer alternatives require complete packed qualification and owner coordination; preserved SDK2 ranges are not silently narrowed. See docs/guides/identity-did-vc-migration.md. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------------------------- | ------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------ | @@ -487,8 +487,8 @@ CLI entry points: `{"lch":"./dist/cli.js"}`. - Package documentation: [docs/packages/helpers/wallet-helper.md](../packages/helpers/wallet-helper.md) - Source: [packages/helpers/bsv-wallet-helper](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/helpers/bsv-wallet-helper) -- Release note: Corrects caller-owned bilateral address derivation and hardens address generation, transaction building, prevout binding, P2PKH, Ordinal, OrdLock, preimage, sighash, OP_RETURN, script, amount, output, and final-wallet-transaction validation. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. -- Migration: getAddress now derives with forSelf: true and returns the caller-owned side of the bilateral relationship. Applications that stored or coordinated the previous peer-owned result must regenerate and exchange the corrected address before sending value. Amount must be an integer from 1 through 1,000 and counterparties must be valid public keys. Valid canonical transaction-builder flows remain supported; malformed, ambiguous, or wallet-mutated results now fail closed. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. +- Release note: Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide. +- Migration: No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative. | Public subpath | Runtime target(s) | Declaration target(s) | | -------------- | ---------------------------------------- | -------------------------------------------- | @@ -514,8 +514,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox.md](../packages/wallet/wallet-toolbox.md) - Source: [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) -- Release note: WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor. Public Argon2id declarations reference the typed hash-wasm package while retaining the same generic binary/string contract and emitted runtime bytes. -- Migration: No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. +- Release note: Adds internal provider-owned journal floor/collection maintenance with complete generation validation, configured migration-owner exclusion, bounded cancellation/result lifetime and physical cleanup admission fencing. Adds internal monotonic continuity-floor transactions and primary-key tombstone collection bounded by examined records, with live-receipt pins and native WAL/RC/RR partial-delete/commit recovery proof. Adds bounded exact source-prefix receipts and explicit immutable receipt policy to owned SQLite/MySQL journal generations, with nonwaiting current reads, bounded expiry collection and process-loss recovery checks. Adds indexed exact SQLite journal source probes and an explicit durable bootstrap row allowance with atomic cursor/charge recovery and source-preserving exhaustion. Adds internal unadvertised SQL journal primitives and source-bound crash-resumable generation ownership, retaining bounded exact-revision pages and bootstrap. Accepts valid terminal all-zero checkpoint resets on unchanged backups while preserving state binding, timestamp monotonicity and nonterminal/partial-reset validation. Adds capability-gated resumable local sync with atomic page/checkpoint commits, cancellation, progress timings, fair foreground scheduling, provider/generation fencing and separated fixed/marginal page-cost fitting. Preserves returned/serialized sync failures and checks chain compatibility before writes. Repairs stale selected/input/broadcast proofs against canonical headers and paths, commits proof corrections with compare-and-set, and fences concurrent monitor repairs. Optimizes unfiltered IndexedDB label offsets without a schema change. Reduces redundant storage promise forwarding while retaining authorization inside exclusive ownership and preserving rejection and queue-release behavior. Shares the remote forwarding rejection boundary and avoids uncontended priority searches and repeated queue/checkpoint helper allocations without changing custom-transport rejection, wire, result or fairness contracts. Shares reader/writer admission helpers without moving authorization across the queue boundary, and uses a Map of property descriptors for in-place entity normalization without an intermediate mapping array. Date, null, byte, record-identity and non-enumerable own-field contracts are retained. Includes the 2.14.4 BRC-100 compatibility fixes: bounded certifier descriptions, requested discovery pagination, signed no-send change outpoints, originator-bound no-send aborts, and receiver-free browser fetch calls in chaintracker clients. WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Runs sequential storage and proof operations through a lazy asynchronous producer while retaining the eight-worker proof bound, failure draining, deterministic transaction order and Promise rejection contracts. Operations are pulled on demand rather than queued eagerly. Captures portable BRC-38 source metadata and all table reads in one provider read view and normalizes absent legacy JSON object fields without mutating source records or dropping array entries. Adds local retained SQLite/MySQL read views with bounded lifetime, single-read admission and physical cleanup ownership; these are excluded from RPC. Adds the version-one local wallet read-snapshot contract; SQL providers deliver profile-bound keyset pages with packed binary columns and payload-size preflight before fetching rows. Integrates supported ordinary local SQL push/pull/backup with a dedicated source reader and atomic durable destination sessions, normalized ID mappings and primary-epoch fencing. Adds syncToWriterResumable and exported local capability/checkpoint types; browser/mobile type availability does not add retained database or remote adapters. Push and backup preserve stored source primary metadata while pull retains the destination selection, including serialized fallback with an older manager cache. Adds internal shared SQL snapshot staging with immutable completed pages, exact retry receipts, profile isolation, bounded logical reservations and cleanup recovery. Adds a local SQL capture controller that binds original metadata/schema to one retained view, checks profile relationships, stores all thirteen raw tables in bounded binary frames and cleans up cancelled or failed captures. Adds bounded metadata-only receipt directories and source/profile/root validation for arbitrary-table page reads without fetching preceding payloads. Exact binding preimages and complete thirteen-table inclusion are preserved. Adds an internal durable creation-request lifecycle with deadline-bound request IDs, shared capacity admission, atomic archive assignment/ready publication and bounded terminal receipt retention. Integrates its capture controller with the provider-owned reader slot, reserves before pool acquisition, drains cancellation/shutdown and preserves ready archives across controller replacement. Physical cleanup failures retain reservations and fence admission, including source-opening failures. Adds a negotiated authenticated snapshot archive transport with prompt durable admission, exact profile-bound status/directory/page/cancel methods, client receipt/root validation and a dedicated two-MiB pre-parse response limit. Expiry reaping precedes admission; resource-limit terminal receipts remain distinct. HTTP shutdown awaits physical capture cleanup. This low-level transport does not yet integrate remote rows with ordinary sync/export or complete #544. Adds an unadvertised immutable remote row reader with v2 server-issued offers, require-existing admission, bounded same-request recovery, fixed leases and packed table paging. Source-only client adapters integrate with local sync through optional provider capabilities; archive positions are detached, checked and atomically persisted. Failure receipts remain observable until cancellation, and typed cancellation preserves unrelated errors. Server reader advertisement stays disabled pending cross-replica owner/physical-cleanup qualification. Preserves the public hash-wasm Argon2id generic contract in emitted declarations without exposing the untyped internal bundle. Adds a durable exact-claim source-owner fence: cross-controller cancellation and both request/archive cleanup retain quota until awaited source cleanup is acknowledged, and page append checks the request fence atomically. Adds fixed-slot backend guards and exact-owner recovery after physical connection closure on local SQLite WAL and same-server MySQL. Old unguarded owners remain reserved and changed backend identities fail closed; reader advertisement stays disabled pending complete qualification. Validates exact request-bound pending-cleanup receipts and polls cancellation with one fixed 30-second cleanup allowance, at most 32 attempts and bounded backoff, without renewing the source lease or hiding independent failures. Aborted I/O settles before cleanup returns. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor. Public Argon2id declarations reference the typed hash-wasm package while retaining the same generic binary/string contract and emitted runtime bytes. Keeps the provider source slot fenced after unproved native cleanup, even after its read promise settles; ordinary read failures remain retryable after proved cleanup. Adds auxiliary profile-key indexes for eight direct SQL tables without changing standard indexes or legacy OFFSET ordering; source triggers and resumable bootstrap keep membership coherent, and retained readers select the complete migration in their existing view. Adds auxiliary numeric relation keys for label/tag mappings with independent parent ownership bases, resumable composite bootstrap, retained-view selection and bounded MySQL primary-index page seeks before and after optimizer statistics refresh. Standard indexes, legacy OFFSET order and cursor/archive encodings remain unchanged. Adds resumable certificate-field indexes preserving source collation, exact text and independent direct/parent ownership, adopted by ordinary and archive readers inside their retained view. Adds resumable global proof/request reference indexes with exact counts, proof-presence guards and retained ordinary/archive selection; standard rows/indexes and BRC-38/cursor bytes are preserved. Adds conflict-safe SQLite auxiliary generations with metadata-bound displaced-identity witnesses, twelve bounded resumable bootstrap streams and bounded retirement of obsolete auxiliary rows. Reader adoption is pinned to journaled completion; MySQL maintenance and standard source records remain unchanged. Legacy SQLite retirement ownership binds object type and trigger name, refusing foreign views that share a trigger name before auxiliary data is retired. SQLite generation publication checks the configured migration-journal schema explicitly, preserving attached-schema journals and rejecting unrelated same-name publication records. Reconciles main’s Postgres storage support while retaining its standard schema and legacy sync path; the current snapshot auxiliary migrations and capabilities remain limited to SQLite/MySQL. Adds internal owned journal capture: reserve two native pools before a nonwaiting writer fence, pin complete generation/profile/schema, commit the exact receipt and verify closure after releasing the barrier. Admission remains held through physical cleanup; an unproved close fences further sources. Preserves the original source/receipt failure alongside every rollback failure when owned journal transactions cannot prove cleanup; admission remains fenced until physical cleanup. Preserves queued primary reselection by checking no-op selections under existing ownership. A pending B switch followed by an A selection finishes with A, including after the earlier failure. A progress formatter failure before transition does not invalidate an in-flight sync. Invalidates cached primary authorization and reloads persisted selection after partial propagation failure; conflicting stores refuse active authorization, and recovery reload failures release queued ownership for retry. Allows exact proof and sync-checkpoint reads in provider-owned MySQL read-only snapshots without requesting write locks, while preserving row locking in ordinary writable transactions. Snapshot classification follows the transaction token across providers and is removed when its callback settles. Adds a complete critical read-snapshot consistency mutation target and generated direct/retained, cross-provider and independent-writer schedules, preserving ordinary unlocked lookups and checkpoint date/boolean normalization without changing the existing source API or isolation policy. +- Migration: Unpublished journal generation helpers now require explicit receipt capacity/lifetime policy and refuse mismatched or earlier unregistered layouts. No registered migration or reader capability is added; capture publication, continuity-floor collection and receiver integration remain unfinished. Internal journal bootstrap callers must supply a fixed row allowance; this unpublished auxiliary layout rejects earlier unregistered generations rather than silently resetting their counters. Registered migration, reset and full resource-retention policy remain unfinished. The internal journal foundation adds no registered migration or public reader advertisement; full quota/capture/floor/receiver integration and full544 acceptance remain incomplete. The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. Use syncFromReaderResumable for the new local atomic-page path; legacy/custom or remote destinations retain exclusive fallback. Cancellation drains in-flight work and resumes from the durable destination checkpoint. Custom providers opt into concurrent reads/atomic pages only when those contracts hold; safe proof persistence additionally requires compareAndSetProvenTxProof. Configured proof services must validate active 80-byte headers. The original legacy page/proof subset needs no migration. The legacy timestamp/offset path may reread equal-time rows and does not provide a coherent source snapshot or streaming archive. Choose maxRoughSize deliberately for device/proxy memory budgets. Current wallet releases remain on their published dependency pins. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Retained views occupy a pool connection and require awaited close/closed cleanup; configure driver deadlines separately. IndexedDB retention remains unsupported. The retained-view primitive introduces no schema or wire migration. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. Apply 2026-09-30-002 add snapshot archive staging through the normal SQL migrate() entry point. Its capacity, manifest and page tables are auxiliary and excluded from BRC-38; no standard-table or legacy-checkpoint change is made. Current staging ceilings are eight handles, 128 MiB reserved globally, 32 MiB per capture, 1 MiB per page, 4096 pages and one hour lifetime. Reservations remain occupied through physical page cleanup. These preliminary limits do not qualify larger-wallet remote export; canonical portable semantic validation and operator resource policy are still required. The local capture controller requires a dedicated reader and an independent staging pool, with schema migrations outside active capture windows; it exposes no RPC or canonical BRC-38 output. Existing sync/backup behavior and browser/mobile adapters are unchanged by this staging component. The internal receipt directory requires no additional schema migration and exposes no RPC capability. Its one-MiB metadata budget does not replace a transport envelope limit; inclusion integrity is separate from portable semantic/proof validation. The existing unpublished 2.15 minor candidate covers these packed core bytes; no package is published by this change. Apply the additive 2026-09-30-003 add snapshot archive requests migration through migrate(); its auxiliary receipts stay outside portable wallet data. Pending requests use the existing archive capacity counters. Close outstanding requests before removing the request schema. The internal controller caps source lifetime at five minutes or the request remaining deadline and keeps the archive deadline fixed. Its per-process reader bound is distinct from shared logical staging quotas; driver cleanup deadlines remain an operator concern. The authenticated archive transport below consumes these internal components; remote row/manager integration and contention performance qualification remain incomplete. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. Apply 2026-10-01-001 add snapshot archive source owners before service capture. Stop and drain captures before downgrading; the migration refuses removal while owners remain and older draft binaries must not capture concurrently. Old unguarded owners retain ownership after unproved process loss. Standard tables and BRC-38/39 bytes are unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Apply 2026-10-01-002 add snapshot archive source guards through migrate(). Preserve the eight persistent slot bindings and SQLite sidecar guard files; never replace a bound file to free quota. New captures require better-sqlite3 WAL or the same actual MySQL server/database. Missing or changed backend identity refuses recovery. Drain all captures before downgrade; old unguarded owners require explicit source cleanup. Distributed filesystems, database relocation/restoration, PXC/load-balanced failover and global physical-connection limits require separate qualification. Reader advertisement stays disabled pending complete lifecycle qualification. After a source-cleanup error, do not reuse the affected provider for snapshot capture; retain its owner reservation until backend recovery proves physical closure. This correction adds no schema or wire change. Apply 2026-10-01-003 add snapshot profile key indexes through migrate(). This explicit nontransactional migration commits bounded 256-row auxiliary key/progress batches and journals only completion; preserve partial objects for retry, verify a terminated migrator before recovering any stale Knex lock, and drain readers before down. Standard rows/indexes and BRC-38/cursor bytes remain unchanged. Relationship/global-table indexing, commit-order high-water and nonblocking IndexedDB remain incomplete. Apply 2026-10-01-004 add snapshot relation key indexes through migrate(). Install/removal observers and producers preserve both ownership bases, including inconsistent mappings for closure refusal. Preserve partial auxiliary DDL/progress for retry, exclude other migrators before lock recovery, and drain readers before down. This migration remains outside BRC-38. Certificate-field/proof indexing, committed-change ordering, streaming and staged import remain incomplete; reader advertisement remains disabled. Apply 2026-10-01-005 add snapshot certificate field key indexes through migrate(). MySQL source and auxiliary tables require InnoDB and matching varchar(100) character set/collation; SQLite verifies the existing complete unique key and built-in BINARY/NOCASE/RTRIM order. Preserve partial owned DDL and committed bootstrap positions, exclude other migrators before verified lock recovery, and drain retained readers before down. Standard tables, indexes, legacy OFFSET and BRC-38/cursor bytes remain unchanged; proof/request indexing and the full program remain incomplete. Apply 2026-10-01-006 add snapshot global reference indexes through migrate(). Its reference edges, exact counts, presence guards and progress are auxiliary and excluded from BRC-38. Preserve partial DDL/committed positions; prove a stopped migrator and exclude others before lock recovery. MySQL requires standard unsigned/text metadata and InnoDB with explicit RESTRICT/NO ACTION mutations; implicit CASCADE/SET NULL actions are refused before DDL. Drain readers before down. The complete journal and progress are adopted within each pinned ordinary/archive view; source rows/indexes, legacy OFFSET and wire bytes stay unchanged. Commit ordering, tombstones, nonblocking IDB, remote destinations, streaming/staged portability and full acceptance remain incomplete; reader advertisement remains off. Apply 2026-10-02-001 repair snapshot SQLite conflict maintenance through migrate(). Preserve partial owned generation/progress for recovery and prove the stopped migrator before stale lock recovery. New readers use source queries while pending and adopt v2 inside the same retained view only after journal publication. Do not downgrade older snapshot binaries against invalidated legacy progress. Ordinary down refuses without deleting source rows; use a forward migration. Explicit dropAllData still deletes the complete wallet, including unpublished generation state. Freed SQLite pages may remain allocated for reuse; bounded retirement does not promise file shrink or full resource-budget acceptance. The SQLite migration-journal lookup correction needs no schema or wire migration; retain the existing tableName/schemaName binding. Postgres records these auxiliary snapshot migrations as no-ops and advertises no retained or paged source. Future Postgres snapshot support needs new forward migrations rather than rewriting the recorded entries. Internal openSnapshotJournalSource requires an existing file-backed better-sqlite3 WAL database or static mysql2 connection and an already complete explicitly owned journal generation. It registers no migration or incremental capability. A killed read view must be captured anew; retained prefix receipts do not reopen it. Runtime quotas, registered journal lifecycle, delta paging and receiver integration remain pending. Internal maintainSnapshotJournal owns generation validation and configured Knex migration exclusion before each bounded floor/collection transaction. Raw operator DDL remains outside that exclusion contract. Cancellation keeps shared capture admission until rollback or committed-result recovery and physical pool cleanup; unproved cleanup fences retained admission and remains observable through destruction. Runtime quotas, registered journal lifecycle/recovery and incremental reader advertisement remain unfinished; no new registered migration or public wire format is introduced. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | ---------------------------------------------------- | -------------------------- | @@ -528,8 +528,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-client.md](../packages/wallet/wallet-toolbox-client.md) - Source: [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) -- Release note: WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. -- Migration: No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. +- Release note: Accepts valid terminal all-zero checkpoint resets on unchanged backups while preserving state binding, timestamp monotonicity and nonterminal/partial-reset validation. Carries the compatible Toolbox resumable sync, fair storage scheduling, canonical proof recovery and IndexedDB paging improvements into the browser entry point. Adds native Chromium large-copy/foreground-latency acceptance alongside the packed browser contract. Reduces redundant storage promise forwarding while retaining authorization inside exclusive ownership and preserving rejection and queue-release behavior. Shares the remote forwarding rejection boundary and avoids uncontended priority searches and repeated queue/checkpoint helper allocations without changing custom-transport rejection, wire, result or fairness contracts. Shares reader/writer admission helpers without moving authorization across the queue boundary, and uses a Map of property descriptors for in-place entity normalization without an intermediate mapping array. Date, null, byte, record-identity and non-enumerable own-field contracts are retained. Includes the 2.14.4 BRC-100 compatibility fixes: bounded certifier descriptions, requested discovery pagination, signed no-send change outpoints, originator-bound no-send aborts, and receiver-free browser fetch calls in chaintracker clients. WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Captures portable BRC-38 source metadata and all table reads in one provider read view and normalizes absent legacy JSON object fields without mutating source records or dropping array entries. Exposes retained-view option/result types and the unsupported base-provider capability without adding a retained IndexedDB or remote implementation. Adds the version-one local wallet read-snapshot contract; SQL providers deliver profile-bound keyset pages with packed binary columns and payload-size preflight before fetching rows. Integrates supported ordinary local SQL push/pull/backup with a dedicated source reader and atomic durable destination sessions, normalized ID mappings and primary-epoch fencing. Adds syncToWriterResumable and exported local capability/checkpoint types; browser/mobile type availability does not add retained database or remote adapters. Push and backup preserve stored source primary metadata while pull retains the destination selection, including serialized fallback with an older manager cache. Adds a negotiated authenticated snapshot archive transport with prompt durable admission, exact profile-bound status/directory/page/cancel methods, client receipt/root validation and a dedicated two-MiB pre-parse response limit. Expiry reaping precedes admission; resource-limit terminal receipts remain distinct. HTTP shutdown awaits physical capture cleanup. This low-level transport does not yet integrate remote rows with ordinary sync/export or complete #544. Adds an unadvertised immutable remote row reader with v2 server-issued offers, require-existing admission, bounded same-request recovery, fixed leases and packed table paging. Source-only client adapters integrate with local sync through optional provider capabilities; archive positions are detached, checked and atomically persisted. Failure receipts remain observable until cancellation, and typed cancellation preserves unrelated errors. Server reader advertisement stays disabled pending cross-replica owner/physical-cleanup qualification. Preserves the public hash-wasm Argon2id generic contract in emitted declarations without exposing the untyped internal bundle. Validates exact request-bound pending-cleanup receipts and polls cancellation with one fixed 30-second cleanup allowance, at most 32 attempts and bounded backoff, without renewing the source lease or hiding independent failures. Aborted I/O settles before cleanup returns. Preserves queued primary reselection by checking no-op selections under existing ownership. A pending B switch followed by an A selection finishes with A, including after the earlier failure. A progress formatter failure before transition does not invalidate an in-flight sync. Invalidates cached primary authorization and reloads persisted selection after partial propagation failure; conflicting stores refuse active authorization, and recovery reload failures release queued ownership for retry. +- Migration: The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. Use the additive syncFromReaderResumable API and its progress/cancellation contracts; custom providers keep the exclusive fallback until they implement the capability contract. No IndexedDB schema migration is required. Source timestamps remain inclusive rather than a coherent snapshot; equal-time boundary rows can be reread. Select explicit page bytes for constrained devices. Publication and downstream wallet adoption are separate actions. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Do not assume retained-view support from the types; inspect the concrete local provider capability. Existing scoped capture remains unchanged. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Backend-bound orphan recovery remains required before reader advertisement. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | @@ -540,8 +540,8 @@ CLI entry points: `{"wallet-relay":"./bin/init.mjs"}`. - Package documentation: [docs/packages/wallet/wallet-toolbox-mobile.md](../packages/wallet/wallet-toolbox-mobile.md) - Source: [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) -- Release note: WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor. -- Migration: No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. +- Release note: Accepts valid terminal all-zero checkpoint resets on unchanged backups while preserving state binding, timestamp monotonicity and nonterminal/partial-reset validation. Carries the compatible Toolbox sync contracts, fair scheduling and canonical proof recovery into the mobile entry point without adding Node or IndexedDB dependencies. Reduces redundant storage promise forwarding while retaining authorization inside exclusive ownership and preserving rejection and queue-release behavior. Shares the remote forwarding rejection boundary and avoids uncontended priority searches and repeated queue/checkpoint helper allocations without changing custom-transport rejection, wire, result or fairness contracts. Shares reader/writer admission helpers without moving authorization across the queue boundary, and uses a Map of property descriptors for in-place entity normalization without an intermediate mapping array. Date, null, byte, record-identity and non-enumerable own-field contracts are retained. Includes the 2.14.4 BRC-100 compatibility fixes: bounded certifier descriptions, requested discovery pagination, signed no-send change outpoints, originator-bound no-send aborts, and receiver-free browser fetch calls in chaintracker clients. WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Captures portable BRC-38 source metadata and all table reads in one provider read view and normalizes absent legacy JSON object fields without mutating source records or dropping array entries. Exposes retained-view option/result types and the unsupported base-provider capability without adding a retained IndexedDB or remote implementation. Adds the version-one local wallet read-snapshot contract; SQL providers deliver profile-bound keyset pages with packed binary columns and payload-size preflight before fetching rows. Integrates supported ordinary local SQL push/pull/backup with a dedicated source reader and atomic durable destination sessions, normalized ID mappings and primary-epoch fencing. Adds syncToWriterResumable and exported local capability/checkpoint types; browser/mobile type availability does not add retained database or remote adapters. Push and backup preserve stored source primary metadata while pull retains the destination selection, including serialized fallback with an older manager cache. Adds a negotiated authenticated snapshot archive transport with prompt durable admission, exact profile-bound status/directory/page/cancel methods, client receipt/root validation and a dedicated two-MiB pre-parse response limit. Expiry reaping precedes admission; resource-limit terminal receipts remain distinct. HTTP shutdown awaits physical capture cleanup. This low-level transport does not yet integrate remote rows with ordinary sync/export or complete #544. Adds an unadvertised immutable remote row reader with v2 server-issued offers, require-existing admission, bounded same-request recovery, fixed leases and packed table paging. Source-only client adapters integrate with local sync through optional provider capabilities; archive positions are detached, checked and atomically persisted. Failure receipts remain observable until cancellation, and typed cancellation preserves unrelated errors. Server reader advertisement stays disabled pending cross-replica owner/physical-cleanup qualification. Preserves the public hash-wasm Argon2id generic contract in emitted declarations without exposing the untyped internal bundle. Validates exact request-bound pending-cleanup receipts and polls cancellation with one fixed 30-second cleanup allowance, at most 32 attempts and bounded backoff, without renewing the source lease or hiding independent failures. Aborted I/O settles before cleanup returns. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor. Preserves queued primary reselection by checking no-op selections under existing ownership. A pending B switch followed by an A selection finishes with A, including after the earlier failure. A progress formatter failure before transition does not invalidate an in-flight sync. Invalidates cached primary authorization and reloads persisted selection after partial propagation failure; conflicting stores refuse active authorization, and recovery reload failures release queued ownership for retry. +- Migration: The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. The additive syncFromReaderResumable API uses atomic local pages only for capable local providers; remote/custom destinations keep the serialized fallback. Existing mobile runtime/crypto backend registration and RPC wires remain unchanged. The original legacy page subset needs no migration. Reconcile cancellation and lost acknowledgements using the durable checkpoint; do not infer coherent source snapshots. Current wallet releases are not required to consume this candidate. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Do not assume retained-view support from the types; inspect the concrete local provider capability. Existing scoped capture remains unchanged. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Backend-bound orphan recovery remains required before reader advertisement. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation. | Public subpath | Runtime target(s) | Declaration target(s) | | ---------------- | -------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------- | diff --git a/docs/reference/stack-facts.md b/docs/reference/stack-facts.md index 5b02fe4a4..7db54015d 100644 --- a/docs/reference/stack-facts.md +++ b/docs/reference/stack-facts.md @@ -52,8 +52,8 @@ authorized release action. | messaging | `@bsv/paymail` | `2.4.10` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/messaging/ts-paymail](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/messaging/ts-paymail) | | middleware | `@bsv/402-pay` | `0.3.3` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/middleware/402-pay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/402-pay) | | middleware | `@bsv/auth` | `0.1.6` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/auth](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth) | -| middleware | `@bsv/auth-express-middleware` | `2.2.9` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/auth-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware) | -| middleware | `@bsv/payment-express-middleware` | `2.1.9` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/payment-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/payment-express-middleware) | +| middleware | `@bsv/auth-express-middleware` | `2.3.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/auth-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/auth-express-middleware) | +| middleware | `@bsv/payment-express-middleware` | `2.2.0` | node-library | node-cjs, node-esm | node | `>=22` | [packages/middleware/payment-express-middleware](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/payment-express-middleware) | | network | `@bsv/chirp` | `0.1.4` | browser-library | browser-bundler, browser-esm, cli, node-esm | browser, node | `>=22` | [packages/network/chirp](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/chirp) | | network | `@bsv/teranode-listener` | `1.1.7` | node-library | node-esm | node | `>=22` | [packages/network/ts-p2p](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/network/ts-p2p) | | overlays | `@bsv/gasp` | `1.3.8` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/overlays/gasp-core](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/overlays/gasp-core) | @@ -67,9 +67,9 @@ authorized release action. | wallet | `@bsv/btms-permission-module` | `1.2.2` | node-library | node-esm | node | `>=22` | [packages/wallet/btms-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/btms-permission-module) | | wallet | `@bsv/ecpm-permission-module` | `0.1.2` | browser-library | browser-bundler, browser-esm, node-esm | browser, node | `>=22` | [packages/wallet/ecpm-permission-module](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ecpm-permission-module) | | wallet | `@bsv/wallet-relay` | `0.5.2` | cli-library | browser-bundler, browser-esm, cli, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/ts-wallet-relay](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/ts-wallet-relay) | -| wallet | `@bsv/wallet-toolbox` | `2.14.5` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | -| wallet | `@bsv/wallet-toolbox-client` | `2.14.5` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | -| wallet | `@bsv/wallet-toolbox-mobile` | `2.14.5` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | +| wallet | `@bsv/wallet-toolbox` | `2.15.0` | node-library | node-cjs | node | `>=22` | [packages/wallet/wallet-toolbox](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox) | +| wallet | `@bsv/wallet-toolbox-client` | `2.15.0` | browser-library | browser-bundler, browser-esm, node-cjs, node-esm | browser, node | `>=22` | [packages/wallet/wallet-toolbox/client](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/client) | +| wallet | `@bsv/wallet-toolbox-mobile` | `2.15.0` | react-native-library | react-native-metro | react-native | `>=22` | [packages/wallet/wallet-toolbox/mobile](https://github.com/bsv-blockchain/ts-stack/tree/main/packages/wallet/wallet-toolbox/mobile) | ## Standalone infrastructure manifests @@ -105,14 +105,14 @@ recorded container release route; they are not published by the public-package j | Metric | Current value | | --- | --- | -| Vector files | 77 | -| Vectors | 6700 | -| Structurally passed | 6489 | +| Vector files | 78 | +| Vectors | 6706 | +| Structurally passed | 6495 | | Governed skips | 211 | -| Required parity vectors | 6496 | +| Required parity vectors | 6502 | | Intended parity vectors | 204 | | Explicitly skipped vector entries | 7 | -| Corpus metadata revision | 2026-09-24 | +| Corpus metadata revision | 2026-09-28 | Structural runner pass/skip results and parity classifications answer different questions: the former is the current runner outcome, while the latter records cross-language diff --git a/docs/reference/test-quality-governance.md b/docs/reference/test-quality-governance.md index 1b26dd1db..825f2e0e6 100644 --- a/docs/reference/test-quality-governance.md +++ b/docs/reference/test-quality-governance.md @@ -147,8 +147,8 @@ when an arbitrary-input invariant can be stated. ## Mutation-validated fuzzing Property generation is only valuable when its assertions can detect a broken -invariant. Every one of the 25 registered property suites therefore has a -matching Stryker mutation target. The target mutates the implementation owned +invariant. Each registered property suite therefore has a matching Stryker mutation +target. The target mutates the implementation owned by that property boundary and runs the smallest relevant combination of property and deterministic regression tests. This catches weak round trips, uncorrelated generators, assertions that only prove “did not throw,” and rare @@ -180,7 +180,7 @@ fan out to all targets; selector, scoring, unrelated SDK, CI, or governance edits do not. Selector and score evaluation are covered by the zero-install repository contract. The independent `Mutation quality` workflow runs the full matrix every Sunday and can run one exact target manually. Targets execute -in parallel, reuse one workspace build, cancel unfinished siblings after a +in parallel, reuse one workspace build, let selected siblings finish after a failure, and preserve machine-readable reports for 30 days. Mutation runs use the policy's fixed 300-case fast-check seed by default so the dry run and every mutant see the same generated campaign. `FAST_CHECK_NUM_RUNS`, @@ -192,11 +192,105 @@ browser/mobile consumers, infrastructure, and runtime images. Empty image and infrastructure matrices do not allocate build runners. The standalone TypeScript conformance workflow runs only when its vectors, specifications, generator, or workflow change; SDK-dependent conformance behavior remains an -affected workspace regression. Cheap repository, dependency, scope, and Sonar -checks gate installation and compilation, matrix lanes cancel siblings on a -failure, and every CI job has a reviewed timeout instead of GitHub's six-hour -default. The zero-install orchestration tests enforce these resource and -fail-fast controls. +affected workspace regression. Cheap repository, dependency and scope checks gate installation and +compilation; Sonar remains required by the final merge gate. Selected matrix +lanes finish after a sibling failure, and every CI job has a reviewed timeout +instead of GitHub's six-hour default. The zero-install orchestration tests +enforce these resource and complete-campaign controls. + +The retained-snapshot target recycles each Stryker test worker after eight mutant +executions to bound accumulated worker state. Every replacement worker runs the +same canonical test selection; the complete mutant union, four-worker concurrency, +property seeds and budgets, per-test limits and aggregate gates remain unchanged. +Worker exits and incomplete reports remain failed qualification evidence. + +The retained-snapshot campaign and the three snapshot-sync groups have a +90-minute limit in PR CI and the standalone mutation workflow. The retained +campaign completed within that allowance, but the complete snapshot-sync campaign +still exceeded it ([hosted timeout](https://github.com/bsv-blockchain/ts-stack/actions/runs/36721037438/job/109907026722)). +Snapshot sync is therefore divided into orchestration (`wallet-snapshot-sync`), +destination (`wallet-snapshot-sync-destination`) and row mapping +(`wallet-snapshot-sync-rows`). Their disjoint ranges preserve the entire original +source scope, and every group runs the complete original selected test suite. +Each group independently requires at least 90% detection and zero uncovered or +invalid mutants; an aggregate score cannot hide a weak group. + +The remote HTTP and service campaigns also exceeded their existing 90-minute +allowances on `6814ed282e617a7b572899915648fe03cf3d47f2`: the +[HTTP job](https://github.com/bsv-blockchain/ts-stack/actions/runs/36854954813/job/110346098383) +started 694 mutants after 550 passing dry-run tests, and the +[service job](https://github.com/bsv-blockchain/ts-stack/actions/runs/36854954813/job/110346098403) +started 1,357 mutants after 266 passing dry-run tests. Neither produced a complete +report. Their execution partitions keep every canonical specification exactly +once. HTTP places all server ranges together, all client ranges together, and +the whole protocol/RPC/transport sources in the protocol fallback. Service places +the whole controller in one part, guard/registry in another, the whole guard +backend separately, and all other sources in the persistence fallback. +Future canonical files join the +fallback automatically. Every execution part keeps the full original tests, +property suite, input dependencies and runner configuration. The existing +provenance and aggregate checks require all parts from the same source and +configuration, then evaluate the complete canonical mutant union; scores are +not averaged. These execution parts do not create new canonical targets. + +The [next complete run](https://github.com/bsv-blockchain/ts-stack/actions/runs/36874901071) +also reached the 90-minute limit for retained reader/lifecycle, archive, remote +reader and service guard execution. Retained profile and relation migrations now +each execute as a whole-file part, alongside reader, storage and the lifecycle +fallback. The subsequent certificate-field migration is registered as another +complete source and whole-file part with the same full retained test selection; +its independently defined fixture is included in the input digest. Global +proof/request migration, model, MySQL metadata, SQLite metadata, bootstrap and +trigger sources are each registered in full, with the independent global fixture +included in the same retained input digest. The entry point and model share +`global-index`; the metadata backends, bootstrap and triggers each run in a +separate whole-file part. This follows a complete local monolithic run lasting +64 minutes that failed the zero-invalid gate; that failed evidence is retained. +The extraction preserves the original declaration bodies, and the separately +tested descriptor/SQL-binding corrections make broken variants fail within the +awaited migration. Every part retains the complete canonical test selection and +the existing 90-minute limit. These parts preserve the complete canonical +source and test union. Archive groups store/migration, source/closure and the capture fallback. +Remote reader groups lease, rows, page/open/cursor and the admission fallback. +These partitions retain every original source specification and full test +configuration. The single-file retained reader remains one complete part; +its traversal tests assert fixture bounds and cursor progress so broken paging +fails promptly. Partitioning and test changes require fresh complete evidence; +the cancelled run does not qualify these targets. + +The eight SQLite conflict-repair helpers remain complete canonical retained +sources. Identity and membership each execute as a whole-file part; generation +shares its part with legacy ownership, bootstrap with state, and retirement with +the migration entry point. The identity/membership split follows a local +90-minute timeout on source `18748e2f4`, with 465 instrumented mutants and all 759 +baseline tests passing but no complete mutation result. Preserve that failure +and its three worker crash warnings. All parts keep the full current canonical +tests and fixtures, four workers, reuse eight, the 90-minute limit, and their +independent critical score and zero-uncovered/invalid gates. Aggregation requires +the complete disjoint source union and fresh matching provenance; no target, +property budget, dependency-selection rule or acceptance threshold changes. + +The journal target includes the complete receipt module in its own execution +part. All twelve journal parts retain the common complete tests/fixtures and +the existing governed property suite, including generated receipt histories. +The native receipt fixture is an explicit additional input and a 60-second group +alongside the unchanged owned MySQL generation/server-crash groups. Target and +property counts, 90% aggregate gate, zero invalid/uncovered requirement, 300-case +seeded property settings, worker/reuse limits and existing deadlines are preserved. + +PR CI downloads each selected target's complete partition artifacts before +canonical verification. The orchestration regression derives the required +downloads from the canonical target registry and execution map, preventing a +new partitioned target from being omitted. Scheduled/manual qualification uses +its existing target matrix to download and independently verify the same union. + +Both workflows use a 45-minute default and the same explicit 90-minute target +allowance list, including retained snapshots, snapshot sync, archive, remote +HTTP, remote reader and remote service. The execution split does not change +those limits, four mutation workers, six parallel jobs, individual mutant/test +deadlines, the 300-case fixed-seed property campaign or the 90% detection and +zero-uncovered/zero-invalid gates. A deadline cancellation is not a completed +report or a passing score. List and run targets locally: diff --git a/governance/dependency-release-policy.json b/governance/dependency-release-policy.json index f551b2359..c31fffbef 100644 --- a/governance/dependency-release-policy.json +++ b/governance/dependency-release-policy.json @@ -1,6 +1,6 @@ { "schemaVersion": 1, - "lastReviewed": "2026-10-01", + "lastReviewed": "2026-10-03", "owner": "ts-stack-maintainers", "routineUpdates": { "dependabotConfig": ".github/dependabot.yml", @@ -144,11 +144,11 @@ "closeImplementationWaveAfterPublication": false }, "overrideRemovalReview": { - "reviewedAt": "2026-10-01", - "method": "Rechecked every registered substitution after the workspace brace-expansion floor moved from 5.0.9 to 5.0.11 and engine.io 6.6.10 was added. Jest and Stryker minimatch still admit brace-expansion below 5.0.11, and socket.io in authsocket still admits engine.io below 6.6.10. Both substitutions stay on the first release that clears the current high advisories. The other 23 selectors, including the Metro-scoped image-size 2.0.4 substitution, remain necessary against the frozen graph. On 2026-10-02 the lodash-es 4.18.1 selector was added because Mermaid 12 brings chevrotain 11.1.2, which pins vulnerable lodash-es 4.17.23. The yamux-scoped @libp2p/utils 7.4.1 selector was added so YamuxStream satisfies @libp2p/interface 3.3 readableEnded. Metro 0.87.1 replaced its image-size dependency with an in-tree parser, so the Metro-scoped image-size substitution was retired on 2026-10-02.", - "retainedCount": 26, - "result": "All 26 retained overrides prevent a reproduced vulnerable transitive version, keep a stream type assignable, or preserve an isolated reproducible toolchain closure. The workspace brace-expansion selector is now 5.0.11, covering GHSA-6j4f-fj2g-mc7p and GHSA-qhr7-859c-m2p7. The engine.io 6.6.10 selector closes GHSA-2gc4-cqfq-p2gv on the authsocket socket.io path without an advisory exclusion. The lodash-es 4.18.1 selector closes GHSA-r5fr-rjxr-66jc on the Mermaid/chevrotain path. The yamux-scoped @libp2p/utils 7.4.1 selector supplies readableEnded for interface 3.3.", - "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch/brace-expansion/js-yaml, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO/engine.io, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, image-size, or Mermaid/chevrotain/lodash-es dependency changes." + "reviewedAt": "2026-10-03", + "method": "Rechecked every registered substitution after the workspace brace-expansion floor moved from 5.0.9 to 5.0.12 and engine.io 6.6.10 was added. Jest and Stryker minimatch still admit brace-expansion below 5.0.12, and socket.io in authsocket still admits engine.io below 6.6.10. Both substitutions stay on the first release that clears the current high advisories. The other 23 selectors, including the Metro-scoped image-size 2.0.4 substitution, remain necessary against the frozen graph. On 2026-10-02 the lodash-es 4.18.1 selector was added because Mermaid 12 brings chevrotain 11.1.2, which pins vulnerable lodash-es 4.17.23. The yamux-scoped @libp2p/utils 7.4.1 selector was added so YamuxStream satisfies @libp2p/interface 3.3 readableEnded. Metro 0.87.1 replaced its image-size dependency with an in-tree parser, so the Metro-scoped image-size substitution was retired on 2026-10-02. Reconciliation with #569 retains its previously qualified 5.0.12 floor for GHSA-q2hr-2g5m-vwhr in both workspace and standalone graphs. On 2026-10-02, no released Metro/file-map/micromatch/braces upgrade removes high GHSA-vfj7-8cjw-p6xm. The exact Metro-file-map0.87.1 watcher patch uses its existing Picomatch2.3.2 some() matcher and removes only scoped micromatch and its unused braces closure; all importers/settings and other package resolutions remain unchanged. On 2026-10-03, three standalone Nodemon-parent Chokidar4.0.3 substitutions remove the affected braces closure. WAB replaces ts-node-dev with the same Nodemon CLI; an owned adapter preserves the old glob-aware ignore predicate and the existing Node/ts-node/telemetry execution contracts. Native restart and shutdown regressions cover each locked service.", + "retainedCount": 30, + "result": "All 30 retained overrides prevent a reproduced vulnerable transitive version, keep a stream type assignable, or preserve an isolated reproducible toolchain closure. The workspace brace-expansion selector is now 5.0.12, covering GHSA-6j4f-fj2g-mc7p and GHSA-qhr7-859c-m2p7. The engine.io 6.6.10 selector closes GHSA-2gc4-cqfq-p2gv on the authsocket socket.io path without an advisory exclusion. The lodash-es 4.18.1 selector closes GHSA-r5fr-rjxr-66jc on the Mermaid/chevrotain path. The yamux-scoped @libp2p/utils 7.4.1 selector supplies readableEnded for interface 3.3. The brace-expansion floor also covers GHSA-q2hr-2g5m-vwhr; no advisory exclusion is introduced. The Metro-file-map exact-version repair removes the affected dependency closure without an audit exclusion; source-owned watcher compatibility, packed Metro/Hermes and complete affected gates remain required. The three scoped Nodemon substitutions remove the affected standalone watcher closure without an advisory exclusion; their source-owned compatibility adapter and complete service/hosted image gates remain required.", + "nextReview": "Rehearse removal monthly and immediately after upstream Google client, Jest/minimatch/brace-expansion/js-yaml, typed-rest-client, Redocly, AJV, express-rate-limit, Socket.IO/engine.io, Vite/PostCSS, remark-mdx-frontmatter/TOML, Metro, image-size, or Mermaid/chevrotain/lodash-es dependency changes. Remove the paired Metro watcher patch, exact package extension and scoped removal together once an official compatible release removes the affected closure and all watcher/platform gates pass. Remove the standalone Nodemon substitutions and adapter together after an official compatible Nodemon release resolves the dependency path natively and all ignore/restart/shutdown, frozen audit, service and Linux image checks pass." }, "overrideRegistry": [ { @@ -159,8 +159,8 @@ }, { "source": "pnpm-workspace.yaml", - "selector": "brace-expansion@<5.0.11", - "value": "5.0.11", + "selector": "brace-expansion@<5.0.12", + "value": "5.0.12", "exceptionId": "brace-expansion-jest-override" }, { @@ -311,6 +311,36 @@ "selector": "gaxios", "value": "7.3.0", "exceptionId": "standalone-gaxios-advisory-override" + }, + { + "source": "pnpm-workspace.yaml", + "selector": "metro-file-map@0.87.1>micromatch", + "value": "'-'", + "exceptionId": "metro-file-map-picomatch-advisory-repair" + }, + { + "source": "infra/uhrp-server-basic/package.json", + "selector": "nodemon", + "value": { + "chokidar": "4.0.3" + }, + "exceptionId": "standalone-nodemon-chokidar-advisory-repair" + }, + { + "source": "infra/uhrp-server-cloud-bucket/package.json", + "selector": "nodemon", + "value": { + "chokidar": "4.0.3" + }, + "exceptionId": "standalone-nodemon-chokidar-advisory-repair" + }, + { + "source": "infra/wab/package.json", + "selector": "nodemon", + "value": { + "chokidar": "4.0.3" + }, + "exceptionId": "standalone-nodemon-chokidar-advisory-repair" } ], "scheduledVerification": { diff --git a/governance/docs-site-bundled-materials.json b/governance/docs-site-bundled-materials.json index 51e1b7525..5042dd087 100644 --- a/governance/docs-site-bundled-materials.json +++ b/governance/docs-site-bundled-materials.json @@ -553,7 +553,7 @@ }, { "name": "dompurify", - "version": "3.4.15", + "version": "3.4.16", "kind": "vite", "packageLicense": "(MPL-2.0 OR Apache-2.0)", "licenseExpression": "Apache-2.0", diff --git a/governance/mutation-testing/REVIEW-2026-10-01.md b/governance/mutation-testing/REVIEW-2026-10-01.md index 5cfece10d..8f0e09768 100644 --- a/governance/mutation-testing/REVIEW-2026-10-01.md +++ b/governance/mutation-testing/REVIEW-2026-10-01.md @@ -66,3 +66,28 @@ funded-side-effect descriptions are adopted independently, without executing a manual or funded suite. The wallet review expires 2026-10-08. The real-date governance check remains required; an inventory review cannot replace exact-source full mutation qualification, successful audits or other required quality gates. + +## Wallet #569 adoption review + +The isolated date patch above is adopted from +`1b711fca31ca1495bc4e11b2d3b3239d15ce7207`. Its two changed policy fields were +independently checked; the 87 baseline manifest/property references exist at the +recorded revision and all 33 main policies match exactly. That source review is +not a claim that the UTXO runtime branch or its target implementations have been +merged into #569. + +On #569, the working reader registry contains 42 actual executable/property +registrations. Its complete prior 41 policy entries, mutation source ranges and +regression-test selections are preserved, with one added HTTP regression file +and the complete ten-file reader target. Every target's manifest, property and +mutation source exists; 172 policy/source/property/workflow input files were +SHA-256-bound in the local review evidence. The new reader retains its critical +90% minimum and zero uncovered/invalid limits. Its mutation run remains pending. +The 338-suite wallet coverage run passes 4,095 cases with the unchanged skip and +782 stable source/test/config hashes; that does not substitute for mutation CI. + +The companion [wallet manual inventory review](../test-quality/wallet-manual-review-2026-10-01.md) +reviews all 30 retained suites without running them and corrects four understated +funded-operation labels. No source/test union, seed, worker, threshold or deadline +is changed by either inventory review. Hosted cancellations remain failures; +subsequent deadline corrections require their own evidence and coordination. diff --git a/governance/mutation-testing/policy.json b/governance/mutation-testing/policy.json index 4973471cf..fe20695c5 100644 --- a/governance/mutation-testing/policy.json +++ b/governance/mutation-testing/policy.json @@ -57,6 +57,106 @@ "maximumNoCoverage": 0, "maximumInvalid": 0 }, + { + "id": "wallet-retained-snapshot", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.property.test.ts", + "risk": "critical", + "boundary": "Wallet retained database-view admission, expiry, cancellation and physical cleanup ownership, profile-bound keyset cursors and bounded packed payload admission", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-journal", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournal.property.test.ts", + "risk": "critical", + "boundary": "Wallet exact journal revisions, bounded metadata pages and bootstrap, full-table observers, source binding and owned crash-resumable generation lifecycle", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-sync", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.property.test.ts", + "risk": "critical", + "boundary": "Wallet durable profile-bound snapshot merges, normalized ID mappings, acknowledged cursors and primary-generation fencing", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-sync-destination", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncDestination.property.test.ts", + "risk": "critical", + "boundary": "Wallet durable snapshot destination binding, atomic checkpoint admission and primary-generation fencing", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-sync-rows", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.property.test.ts", + "risk": "critical", + "boundary": "Wallet bounded snapshot row detachment and profile/source-scoped normalized parent and child ID mapping", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-archive", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts", + "risk": "critical", + "boundary": "Wallet coherent SQL snapshot capture and staging, profile closure, immutable receipts, quotas and cleanup accounting", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-remote-directory", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts", + "risk": "critical", + "boundary": "Wallet authenticated snapshot receipt directories, original binding bytes, complete table hash chains and page inclusion", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-remote-service", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts", + "risk": "critical", + "boundary": "Wallet remote snapshot creation request identity, durable receipt publication, bounded reservation, exact source ownership, backend guards and physical cleanup recovery", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-remote-http", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.property.test.ts", + "risk": "critical", + "boundary": "Wallet authenticated snapshot HTTP negotiation, exact protocol fields, profile binding, immutable client receipts and bounded transport lifecycle", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, + { + "id": "wallet-snapshot-remote-reader", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.property.test.ts", + "risk": "critical", + "boundary": "Wallet immutable remote snapshot row decoding, fixed archive cursors, bounded leases, exact admission recovery and pending cleanup", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 + }, { "id": "overlay-linkage", "manifest": "packages/overlays/topics/package.json", @@ -386,6 +486,16 @@ "minimumScore": 90, "maximumNoCoverage": 0, "maximumInvalid": 0 + }, + { + "id": "wallet-read-snapshot-consistency", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "propertyTest": "packages/wallet/wallet-toolbox/src/storage/portable/mysqlReadSnapshot.property.test.ts", + "risk": "critical", + "boundary": "Wallet callback-owned coherent transaction classification, exact proof/checkpoint consistent reads and preservation of writable authority locks", + "minimumScore": 90, + "maximumNoCoverage": 0, + "maximumInvalid": 0 } ] } diff --git a/governance/mutation-testing/targets.mjs b/governance/mutation-testing/targets.mjs index 654f00b4b..40e5db2c4 100644 --- a/governance/mutation-testing/targets.mjs +++ b/governance/mutation-testing/targets.mjs @@ -17,10 +17,16 @@ function sourceLineRange(repositoryRoot, packageDirectory, filePath, startMarker return `${filePath}:${startIndex + 1}-${endIndex}` } -function jestTarget(configFile, testMatch, { esm = false, config = {}, findRelated = false } = {}) { +function jestTarget( + configFile, + testMatch, + { esm = false, config = {}, findRelated = false, buildCommand, maxTestRunnerReuse } = {} +) { return { testRunner: 'jest', runnerOptions: { + ...(buildCommand ? { buildCommand } : {}), + ...(maxTestRunnerReuse === undefined ? {} : { maxTestRunnerReuse }), jest: { projectType: 'custom', configFile, @@ -51,6 +57,195 @@ function vitestTarget(configFile) { } } +// Preserve the complete original snapshot-sync source ranges and selected tests. +// Each disjoint group runs the same behavioral suite and its own strict gate. +function snapshotSyncMutationTargets(repositoryRoot) { + const complete = { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.property.test.ts', + mutate: [ + 'src/utility/runInSeries.ts', + 'src/storage/sync/syncCheckpoint.ts', + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/sync/syncSession.ts', + 'async function committedCheckpoint(', + '/** One page in flight;' + ), + 'src/storage/snapshot/SnapshotSync.ts', + 'src/storage/snapshot/SnapshotSyncRows.ts', + 'src/storage/snapshot/KnexSnapshotSyncDestination.ts', + 'src/storage/snapshot/runSnapshotSyncSession.ts', + 'src/storage/schema/snapshotSyncMigration.ts', + 'src/storage/schema/entities/mergeSyncChunkEntities.ts', + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/sync/syncSession.ts', + 'if (chunk.user != null && session.activeStorage', + "notify('preparing', { readMs })" + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/methods/validateSyncProof.ts', + 'if (!(candidate.rawTx instanceof Uint8Array', + ' try {' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'this.snapshotSyncEnabled = options.snapshotSync', + 'this.preparedBeefPolicy =' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/schema/entities/EntityProvenTxReq.ts', + 'override async mergeExisting(', + 'export interface ProvenTxReqHistorySummaryApi' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override getSnapshotSync():', + 'protected override supportsNoSendExpiryPersistence()' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override async destroy(): Promise', + 'override async migrate(' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'private async runSnapshotCopy(', + 'async syncFromReader(' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'async syncFromReader(', + ' let inserts = 0' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'async syncFromReaderResumable(', + ' const generation =' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'async syncToWriterResumable(', + 'async syncToWriter(' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'async syncToWriter(', + ' let inserts = 0' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'async updateBackups(', + 'async setActive(' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'async setActive(', + 'getStoreEndpointURL(' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/WalletStorageManager.ts', + 'private async withAccess(', + 'runAsWriter(' + ) + ], + ...jestTarget( + 'jest.config.cjs', + [ + '/src/storage/snapshot/SnapshotSync*.test.ts', + '/src/storage/snapshot/ConcurrentSnapshotSyncSource.test.ts', + '/src/storage/snapshot/ConcurrentSnapshotArchiveSource.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveService.test.ts', + '/src/storage/schema/snapshotSyncMigration.test.ts', + '/src/storage/methods/validateSyncProof.test.ts', + '/src/storage/sync/syncFailure.test.ts', + '/src/storage/sync/syncSession.test.ts', + '/src/storage/sync/syncCheckpoint.test.ts', + '/src/utility/__tests__/runInSeries.test.ts', + '/src/storage/snapshot/journal/*.test.ts' + ], + { + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + } + const destination = new Set([ + 'src/storage/snapshot/KnexSnapshotSyncDestination.ts', + 'src/storage/schema/snapshotSyncMigration.ts' + ]) + const rows = new Set([ + 'src/storage/snapshot/SnapshotSyncRows.ts', + 'src/storage/schema/entities/mergeSyncChunkEntities.ts', + 'src/storage/schema/entities/EntityProvenTxReq.ts', + 'src/storage/methods/validateSyncProof.ts' + ]) + const groups = { + 'wallet-snapshot-sync': [], + 'wallet-snapshot-sync-destination': [], + 'wallet-snapshot-sync-rows': [] + } + const propertyTests = { + 'wallet-snapshot-sync': complete.propertyTest, + 'wallet-snapshot-sync-destination': + 'packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncDestination.property.test.ts', + 'wallet-snapshot-sync-rows': + 'packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.property.test.ts' + } + for (const range of complete.mutate) { + const file = range.replace(/:\d+(?:-\d+)?$/, '') + let name = 'wallet-snapshot-sync' + if (destination.has(file)) name = 'wallet-snapshot-sync-destination' + else if (rows.has(file)) name = 'wallet-snapshot-sync-rows' + groups[name].push(range) + } + return Object.fromEntries( + Object.entries(groups).map(([name, mutate]) => [ + name, + { + ...complete, + mutate, + propertyTest: propertyTests[name] + } + ]) + ) +} + export function buildMutationTargets(repositoryRoot) { return { 'sdk-codecs': { @@ -230,6 +425,485 @@ export function buildMutationTargets(repositoryRoot) { } ) }, + 'wallet-read-snapshot-consistency': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + additionalInputs: [ + 'test/storage/snapshotArchiveMysql.cjs', + 'test/storage/runSnapshotArchiveMysql.cjs', + 'test/storage/snapshotArchiveDocker.cjs', + 'test/storage/snapshotMysqlFixtureGroups.cjs', + 'test/utils/mysqlReadSnapshotFixture.ts' + ], + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/portable/mysqlReadSnapshot.property.test.ts', + mutate: [ + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + '// Only provider-owned snapshot callbacks', + 'interface KnexTelemetryQuery' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override async readSnapshot', + 'override supportsRetainedReadSnapshot(): boolean' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'private async readMySQLSnapshot', + 'override getSnapshotSync():' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override async findProvenTxs(', + 'override async findStaleMerkleRoots(' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override async findSyncStates(', + 'override async findTransactions(' + ) + ], + ...jestTarget( + 'jest.config.cjs', + [ + '/src/storage/portable/snapshot.test.ts', + '/src/storage/portable/mysqlSnapshot.test.ts', + '/src/storage/snapshot/StorageKnex.retainedSnapshot.test.ts', + '/src/storage/snapshot/KnexWalletReadSnapshot.test.ts', + '/src/storage/snapshot/RetainedReadSnapshot.property.test.ts', + '/src/storage/portable/mysqlReadSnapshot.property.test.ts' + ], + { + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + }, + 'wallet-retained-snapshot': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + additionalInputs: [ + 'test/utils/snapshotRelationFixtures.ts', + 'test/utils/snapshotCertificateFixtures.ts', + 'test/utils/snapshotGlobalFixtures.ts', + 'test/utils/snapshotHistoricalMigrations.ts', + 'test/utils/snapshotSqliteFixtures.ts', + 'test/utils/snapshotSqliteIdentityFixture.ts', + 'test/utils/snapshotSqliteMaintenanceFixture.ts', + 'test/storage/snapshotHistoricalMigrations.cjs', + 'test/storage/snapshotSqliteGenerationCrash.cjs' + ], + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.property.test.ts', + mutate: [ + 'src/storage/snapshot/RetainedReadSnapshot.ts', + 'src/storage/snapshot/KnexWalletReadSnapshot.ts', + 'src/storage/schema/snapshotSqlMigration.ts', + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/schema/KnexMigrations.ts', + 'migrations[SNAPSHOT_SQLITE_INDEX_MIGRATION] = {', + 'migrations[SYNC_TRANSFER_MIGRATION] = {' + ), + 'src/storage/schema/snapshotProfileIndexMigration.ts', + 'src/storage/schema/snapshotRelationIndexMigration.ts', + 'src/storage/schema/snapshotCertificateIndexMigration.ts', + 'src/storage/schema/snapshotGlobalIndexMigration.ts', + 'src/storage/schema/snapshotGlobalIndexModel.ts', + 'src/storage/schema/snapshotGlobalIndexMysql.ts', + 'src/storage/schema/snapshotGlobalIndexSqlite.ts', + 'src/storage/schema/snapshotGlobalIndexBootstrap.ts', + 'src/storage/schema/snapshotGlobalIndexTriggers.ts', + 'src/storage/schema/snapshotSqliteIdentity.ts', + 'src/storage/schema/snapshotSqliteMembership.ts', + 'src/storage/schema/snapshotSqliteIndexGeneration.ts', + 'src/storage/schema/snapshotSqliteIndexBootstrap.ts', + 'src/storage/schema/snapshotSqliteIndexState.ts', + 'src/storage/schema/snapshotSqliteIndexRetirement.ts', + 'src/storage/schema/snapshotSqliteLegacyOwnership.ts', + 'src/storage/schema/snapshotSqliteIndexMigration.ts', + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override supportsRetainedReadSnapshot(): boolean', + 'private async readMySQLSnapshot' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'async openSnapshotJournalSource(', + 'recoverSnapshotArchiveSources(): Promise' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override async destroy(): Promise', + 'override async migrate(' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageKnex.ts', + 'override async dropAllData(): Promise', + 'override async transaction' + ), + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + 'src/storage/StorageProvider.ts', + 'supportsRetainedReadSnapshot(): boolean', + 'protected supportsActionBatchPersistence(): boolean' + ) + ], + ...jestTarget( + 'jest.config.cjs', + [ + '/src/storage/snapshot/*.test.ts', + '/src/storage/snapshot/journal/SnapshotJournalCapture*.test.ts', + '/src/storage/snapshot/journal/SnapshotJournalConnections.test.ts', + '/src/storage/snapshot/journal/SnapshotJournalMaintenance*.test.ts', + '/src/storage/__test/StorageKnexMigrationFailure.security.test.ts' + ], + { + maxTestRunnerReuse: 8, + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + }, + 'wallet-snapshot-journal': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + additionalInputs: [ + 'test/fixtures/snapshotJournal/mysql-generation-ddl-fixture.json', + 'test/fixtures/snapshotJournal/mysql-generation-metadata-fixture.json', + 'test/fixtures/snapshotJournal/mysql-generation-state-fixture.json', + 'test/fixtures/snapshotJournal/mysql-intent-metadata-fixture.json', + 'test/fixtures/snapshotJournal/mysql-source-metadata-fixture.json', + 'test/utils/snapshotArchiveFixtures.ts', + 'test/utils/snapshotSqliteFixtures.ts', + 'test/utils/snapshotHistoricalMigrations.ts', + 'test/storage/snapshotJournalNativeFixture.cjs', + 'test/storage/snapshotJournalMysqlConnection.cjs', + 'test/storage/snapshotJournalMysql.cjs', + 'test/storage/snapshotJournalMysqlServerCrash.cjs', + 'test/storage/snapshotJournalReceiptMysql.cjs', + 'test/storage/snapshotJournalCaptureMysql.cjs', + 'test/storage/snapshotJournalCaptureMysqlChild.cjs', + 'test/storage/snapshotJournalCaptureProcessLoss.cjs', + 'test/storage/snapshotJournalCaptureSqlite.cjs', + 'test/storage/snapshotJournalRetentionChild.cjs', + 'test/storage/snapshotJournalRetentionCuts.cjs', + 'test/storage/snapshotJournalRetentionMysql.cjs', + 'test/storage/snapshotJournalRetentionMysqlRc.cjs', + 'test/storage/snapshotJournalRetentionMysqlRr.cjs', + 'test/storage/snapshotJournalRetentionProcessLoss.cjs', + 'test/storage/snapshotJournalRetentionSqlite.cjs', + 'test/storage/snapshotJournalMaintenanceFixture.cjs', + 'test/storage/snapshotJournalMaintenanceCuts.cjs', + 'test/storage/snapshotJournalMaintenanceChild.cjs', + 'test/storage/snapshotJournalMaintenanceProcessLoss.cjs', + 'test/storage/snapshotJournalMaintenanceWal.cjs', + 'test/storage/snapshotJournalMaintenanceMysqlRc.cjs', + 'test/storage/snapshotJournalMaintenanceMysqlRr.cjs', + 'test/storage/snapshotJournalSqliteCrash.cjs', + 'test/storage/runSnapshotJournalMysql.cjs', + 'test/storage/snapshotArchiveDocker.cjs' + ], + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournal.property.test.ts', + mutate: [ + 'src/storage/snapshot/journal/SnapshotJournalRevision.ts', + 'src/storage/snapshot/journal/SnapshotJournalRevisionSql.ts', + 'src/storage/snapshot/journal/SnapshotJournalPage.ts', + 'src/storage/snapshot/journal/SnapshotJournalSqliteClock.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlClock.ts', + 'src/storage/snapshot/journal/SnapshotJournalSqliteObservers.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlObservers.ts', + 'src/storage/snapshot/journal/SnapshotJournalBootstrap.ts', + 'src/storage/snapshot/journal/SnapshotJournalHighWater.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlSource.ts', + 'src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlIntent.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.ts', + 'src/storage/snapshot/journal/SnapshotJournalReceipt.ts', + 'src/storage/snapshot/journal/SnapshotJournalCaptureFence.ts', + 'src/storage/snapshot/journal/SnapshotJournalConnections.ts', + 'src/storage/snapshot/journal/SnapshotJournalCaptureBackend.ts', + 'src/storage/snapshot/journal/SnapshotJournalCapture.ts', + 'src/storage/snapshot/journal/SnapshotJournalCollection.ts', + 'src/storage/snapshot/journal/SnapshotJournalMaintenance.ts', + 'src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.ts', + 'src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.ts' + ], + ...jestTarget('jest.config.cjs', ['/src/storage/snapshot/journal/*.test.ts'], { + maxTestRunnerReuse: 8, + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + }) + }, + 'wallet-snapshot-archive': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts', + mutate: [ + 'src/storage/snapshot/archive/KnexSnapshotArchiveStore.ts', + 'src/storage/snapshot/archive/SnapshotArchive.ts', + 'src/storage/snapshot/archive/SnapshotArchiveSql.ts', + 'src/storage/snapshot/archive/KnexSnapshotArchiveClosure.ts', + 'src/storage/snapshot/archive/KnexSnapshotArchiveSource.ts', + 'src/storage/snapshot/archive/captureKnexSnapshotArchive.ts', + 'src/storage/snapshot/archive/captureSnapshotArchiveSource.ts', + 'src/storage/schema/snapshotArchiveMigration.ts' + ], + ...jestTarget( + 'jest.config.cjs', + [ + '/src/storage/snapshot/archive/KnexSnapshotArchiveStore.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveCapture.test.ts', + '/src/storage/snapshot/archive/SnapshotArchiveDirectory.test.ts', + '/src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts', + '/src/storage/snapshot/ConcurrentSnapshotArchiveSource.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveService.test.ts' + ], + { + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + }, + 'wallet-snapshot-remote-directory': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts', + mutate: ['src/storage/snapshot/archive/SnapshotArchiveDirectory.ts'], + ...jestTarget( + 'jest.config.cjs', + ['/src/storage/snapshot/archive/SnapshotArchiveDirectory*.test.ts'], + { + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + }, + 'wallet-snapshot-remote-service': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts', + mutate: [ + 'src/storage/snapshot/archive/SnapshotArchiveRequest.ts', + 'src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.ts', + 'src/storage/snapshot/archive/KnexSnapshotArchiveService.ts', + 'src/storage/snapshot/archive/SnapshotArchiveSql.ts', + 'src/storage/schema/snapshotArchiveRequestMigration.ts', + 'src/storage/snapshot/archive/SnapshotArchiveOwner.ts', + 'src/storage/schema/snapshotArchiveOwnerMigration.ts', + 'src/storage/schema/snapshotArchiveGuardMigration.ts', + 'src/storage/snapshot/archive/SnapshotArchiveGuard.ts', + 'src/storage/snapshot/archive/SnapshotArchiveGuardRegistry.ts', + 'src/storage/snapshot/archive/SnapshotArchiveGuardBackend.ts' + ], + ...jestTarget( + 'jest.config.cjs', + [ + '/src/storage/snapshot/archive/SnapshotArchiveRequest.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveStore.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveService.test.ts', + '/src/storage/snapshot/archive/SnapshotArchiveGuard*.test.ts', + '/src/storage/snapshot/ConcurrentSnapshotArchiveSource.test.ts', + '/src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts' + ], + { + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + }, + 'wallet-snapshot-remote-http': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.property.test.ts', + mutate: [ + 'src/storage/snapshot/archive/SnapshotArchiveProtocol.ts', + 'src/storage/snapshot/archive/KnexSnapshotArchiveRpc.ts', + 'src/storage/snapshot/archive/SnapshotArchiveTransport.ts', + ...[ + [ + 'src/storage/remoting/StorageClientBase.ts', + 'async processSyncChunk(', + 'async getSyncChunk(' + ], + [ + 'src/storage/remoting/StorageClientBase.ts', + 'if (properties.snapshotArchive != null)', + 'return value as RemoteStorageSettings' + ], + [ + 'src/storage/remoting/StorageClientBase.ts', + 'this.snapshotWallet = wallet', + 'this.endpointUrl =' + ], + [ + 'src/storage/remoting/StorageClientBase.ts', + 'protected async authenticatedFetch(', + 'protected async traceRpcCall(' + ], + [ + 'src/storage/remoting/StorageClientBase.ts', + 'if (settings.snapshotArchive !== undefined', + 'this.settings = settings' + ], + [ + 'src/storage/remoting/StorageServer.ts', + 'private readonly snapshotArchivesEnabled:', + 'private readonly app =' + ], + [ + 'src/storage/remoting/StorageServer.ts', + 'this.snapshotArchivesEnabled =', + '// Keep legacy configurations working' + ], + [ + 'src/storage/remoting/StorageServer.ts', + 'private async handleRpcRequestCore(', + 'private async sendOversizedSyncResponse(' + ], + [ + 'src/storage/remoting/StorageServer.ts', + '...(await this.snapshotArchiveSettings())', + 'this.finishRpcLogging(logger, result)' + ], + [ + 'src/storage/remoting/StorageServer.ts', + 'private async snapshotArchiveSettings(', + 'private async dispatchSyncTransfer(' + ], + [ + 'src/storage/remoting/StorageServer.ts', + 'private createSnapshotArchiveRpc(', + 'private async traceRpcStep(' + ], + ['src/storage/remoting/StorageServer.ts', 'public start(): void', 'validateDate(date:'] + ].map(([filePath, startMarker, endMarker]) => + sourceLineRange( + repositoryRoot, + 'packages/wallet/wallet-toolbox', + filePath, + startMarker, + endMarker + ) + ) + ], + ...jestTarget( + 'jest.config.cjs', + [ + '/src/storage/snapshot/archive/SnapshotArchive*.test.ts', + '/src/storage/snapshot/archive/RemoteSnapshotReaderHttp.test.ts', + '/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.test.ts', + '/src/storage/remoting/__test/BinaryJson.test.ts', + '/src/storage/remoting/__test/KnexPaymentReplayStore.test.ts', + '/src/storage/remoting/__test/KnexSessionManager.test.ts', + '/src/storage/remoting/__test/RateLimitPolicy.test.ts', + '/src/storage/remoting/__test/StorageServerRpc.test.ts', + '/src/storage/remoting/__test/StorageClientBase.*.test.ts', + '/src/storage/sync/syncCheckpoint.test.ts', + '/src/storage/remoting/__test/StorageClient.security.test.ts', + '/src/storage/remoting/__test/StorageClient.transport.security.test.ts', + '/src/storage/remoting/__test/StorageClient.telemetry.test.ts' + ], + { + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + }, + 'wallet-snapshot-remote-reader': { + packageDirectory: 'packages/wallet/wallet-toolbox', + manifest: 'packages/wallet/wallet-toolbox/package.json', + propertyTest: + 'packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.property.test.ts', + mutate: [ + 'src/storage/snapshot/archive/SnapshotArchiveReaderRequest.ts', + 'src/storage/snapshot/archive/SnapshotArchiveReaderOffer.ts', + 'src/storage/snapshot/archive/SnapshotArchiveAdmission.ts', + 'src/storage/snapshot/archive/SnapshotArchiveTransportFailure.ts', + 'src/storage/snapshot/archive/SnapshotArchiveCleanup.ts', + 'src/storage/snapshot/archive/RemoteSnapshotLease.ts', + 'src/storage/snapshot/archive/RemoteSnapshotRows.ts', + 'src/storage/snapshot/archive/RemoteSnapshotPageReader.ts', + 'src/storage/snapshot/archive/openRemoteSnapshot.ts', + 'src/storage/snapshot/SnapshotCursor.ts', + 'src/storage/snapshot/SnapshotCancelledError.ts' + ], + ...jestTarget( + 'jest.config.cjs', + [ + '/src/storage/snapshot/archive/RemoteSnapshot*.test.ts', + '/src/storage/snapshot/archive/SnapshotArchiveReader*.test.ts', + '/src/storage/snapshot/archive/SnapshotArchiveAdmission.test.ts', + '/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.test.ts', + '/src/storage/snapshot/SnapshotCursor.test.ts' + ], + { + config: { + moduleNameMapper: { + '^@bsv/sdk$': resolve(repositoryRoot, 'packages/sdk/mod.ts'), + '^(\\.{1,2}/.*)\\.js$': '$1' + } + } + } + ) + }, + ...snapshotSyncMutationTargets(repositoryRoot), 'overlay-linkage': { packageDirectory: 'packages/overlays/topics', manifest: 'packages/overlays/topics/package.json', @@ -537,10 +1211,10 @@ export function buildMutationTargets(repositoryRoot) { propertyTest: 'packages/middleware/auth-express-middleware/src/__tests/authMiddlewareHelpers.property.test.ts', mutate: [ - 'src/authMiddlewareHelpers.ts:96-103', - 'src/authMiddlewareHelpers.ts:154-163', - 'src/authMiddlewareHelpers.ts:180-184', - 'src/authMiddlewareHelpers.ts:206-233' + 'src/authMiddlewareHelpers.ts:99-114', + 'src/authMiddlewareHelpers.ts:155-164', + 'src/authMiddlewareHelpers.ts:181-185', + 'src/authMiddlewareHelpers.ts:198-238' ], ...jestTarget('jest.config.js', ['/src/__tests/authMiddlewareHelpers*.test.ts']) }, @@ -549,7 +1223,7 @@ export function buildMutationTargets(repositoryRoot) { manifest: 'packages/middleware/payment-express-middleware/package.json', propertyTest: 'packages/middleware/payment-express-middleware/src/__tests/PaymentReplayStore.property.test.ts', - mutate: ['src/index.ts:27-44'], + mutate: ['src/index.ts:30-47'], ...jestTarget('jest.config.js', ['/src/__tests/PaymentReplayStore*.test.ts']) }, 'wallet-script-encoding': { diff --git a/governance/package-release-notes.json b/governance/package-release-notes.json index c47bc8537..7b1087840 100644 --- a/governance/package-release-notes.json +++ b/governance/package-release-notes.json @@ -7,8 +7,8 @@ "name": "@bsv/402-pay", "publishedVersion": "0.3.2", "releaseType": "patch", - "summary": "Adds an exact-tarball Vite and esbuild contract for the browser-safe client entry point, including a bundle-size ratchet and an assertion that server exports never leak into browser consumers. Retains the hash-pinned pre-uniformization Open BSV License version 4 grant as a scoped continuity notice. Standardizes first-party author metadata on the current BSV Association name. Binds BRC-121 pricing and internalization to the transaction declared by the BRC-95 Atomic BEEF subject, removes unrelated included branches, rejects trailing bytes and non-atomic envelopes, and requires affirmative wallet acceptance before serving paid content. Adds independent bounded atomic transaction replay claims so conforming BRC-100 wallets that omit the non-public isMerge detail cannot authorize duplicate access, preserves actual overpayments in middleware receipts, uses fixed bounded wallet descriptions, makes diagnostics opt-in, and returns an unchallenged 503 after ambiguous wallet or replay-store failures to avoid inducing a second spend. Aligns the development-only Vitest runner and V8 coverage provider at 4.1.11. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "Custom clients must send a strictly framed BRC-95 Atomic BEEF envelope whose subject is the payment transaction; plain BEEF is no longer accepted. Legacy Atomic BEEF containing unrelated branches remains compatible because the server reduces it to the declared subject and dependency closure. The default replay store is bounded and process-local; production services with more than one serving process or node must inject the same durable atomic PaymentReplayStore everywhere. Low-level validators should retain one wallet object or pass an explicit store. Applications that relied on implicit console diagnostics must supply the optional structured logger. Treat an unchallenged 503 after payment submission as ambiguous and reconcile the transaction before requesting another payment. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Documents the protocol boundary between this package's BRC-121 header payments and the separate BRC-105/BRC-118 authenticated multipart integration. Runtime and wire behavior are unchanged. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "No migration. Continue using the existing BRC-121 headers and replay store. Select SDK AuthFetch with auth/payment Express middleware for BRC-118; this package does not implicitly negotiate that protocol. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/air-gap", @@ -21,8 +21,8 @@ "name": "@bsv/amountinator", "publishedVersion": "2.1.6", "releaseType": "patch", - "summary": "Validates finite monetary values, normalized nonempty currency codes, integer decimal precision, grouping flags, converter rates, and conversion results, and formats negative sub-unit amounts from their absolute magnitude without losing the sign. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "Valid finite inputs retain the public API. Currency identifiers are trimmed and normalized to uppercase; callers that passed non-finite values, empty currencies, coercive options, invalid decimal precision, or non-finite converter results must normalize or reject them before calling because those values now fail closed. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/auth", @@ -34,9 +34,9 @@ { "name": "@bsv/auth-express-middleware", "publishedVersion": "2.2.8", - "releaseType": "patch", - "summary": "Removes middleware header byte/count ceilings and excludes received HTTP headers from authenticated request-body budgets, preserving signed header bytes. HTTP server and edge layers own transport capacity. Canonical framing, authentication, invalid-header rejection, body budgets and response signing remain. Requires SDK 2.8.5 and delegates its Peer general-payload capacity to the HTTP layer, avoiding an indirect SDK envelope ceiling for received headers. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "No BRC100 call, wire or wallet-data migration. Configure and validate transport header limits at the HTTP server, CDN, proxy or WAF; maxRequestBytes no longer caps received headers. Clients needing larger signed responses require corresponding client capacity, including SDK 2.8.5. Published 2.2.8 is verified against protected release 36052862859 and its immutable source artifacts. Upgrade the required SDK peer to 2.8.5; custom ExpressTransport/Peer setups should pass maxGeneralPayloadBytes=null when the HTTP layer owns payload capacity. SDK consumers may retain ^2.8.5 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "releaseType": "minor", + "summary": "Adds opt-in bounded raw request collection for BRC-118, authenticates the whole body and exact multipart Content-Type before extraction, restores decoded non-multipart application payloads after authentication, and preserves Express set(object) response-header behavior. Normalizes zero-length dense byte arrays and Uint8Array/Buffer bodies to the BRC-104 -1 sentinel, matching AuthFetch when express.raw runs before authentication. Frames typed-array bodies from one copied byte vector, so length metadata and serialized payload cannot disagree. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "Requires @bsv/sdk ^2.9.0 || ^3.0.0. Set captureRawBody:true before body parsers when enabling multipart payment middleware. Default parsed-body integration remains available; nonempty non-multipart preimages remain compatible. Raw collection rejects compressed, oversized, slow and already-consumed bodies and enforces aggregate memory and pending-request bounds. Multipart boundary parameters must be signed exactly; receivers do not fall back to verification without the boundary. Configure public credential-free CORS preflight and expose the negotiation header. No deployment or publication is performed by this source candidate. For empty binary requests, upgrade SDK clients to the integrated 3.0.0 candidate and auth receivers to 2.3.0 together; older length-0 signatures/reconstructions need the paired correction. Nonempty preimages are unchanged. The retained SDK2 alternative starts at ^2.9.0; the integrated source graph now uses SDK3, including the paired BRC-118 correction. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/authsocket", @@ -49,8 +49,8 @@ "name": "@bsv/authsocket-client", "publishedVersion": "2.1.7", "releaseType": "patch", - "summary": "Contains authentication and application callback failures, caps authentication concurrency, snapshots strict JSON before signing, serializes real typed arrays portably, preserves supported signed event JSON exactly, and ships the complete SDK incorporated-material notice archive with a retained UMD notice banner. Standardizes first-party author metadata on the current BSV Association name. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "No API or wire migration is required for valid JSON events. Existing numeric-key objects under byte-like names are unchanged; typed payment protocols recover historical byte objects at their explicit fields. Outbound non-JSON or ambiguous runtime values, including negative zero, nested undefined, sparse arrays, accessors, hidden/extra properties, and serialization hooks, now fail closed before signing. The connect event/connected property report Socket.IO transport state, not completed BRC-103 authentication; wait for verified application traffic when local behavior requires a known unpinned server. Distributors who copy the UMD file must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Refreshes the distributed UMD browser bundle with the integrated SDK implementation. The rebuilt bundle differs from published 2.1.7; the patch candidate keeps those new bytes under a new version. AuthSocket source APIs and event serialization are unchanged. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "No API or event-wire migration. ESM/CommonJS consumers retain their existing SDK peer range. Browser distributors adopt the new UMD artifact after publication and must keep THIRD_PARTY_NOTICES.md and LICENSES/ with it. This source candidate does not publish or update current wallet release pins. SDK consumers may retain ^2.4.1 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/btms", @@ -98,8 +98,8 @@ "name": "@bsv/fund-wallet", "publishedVersion": "1.5.2", "releaseType": "patch", - "summary": "Requires the funding wallet's internalizeAction result to contain accepted: true before reporting a funded transaction as successfully internalized. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "No public API migration is required for conforming wallets. Custom WalletInterface adapters must return the literal accepted: true verdict after successful internalization; refusals and malformed or coercive results now throw instead of being reported as success. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/gasp", @@ -112,8 +112,8 @@ "name": "@bsv/message-box-client", "publishedVersion": "2.5.4", "releaseType": "minor", - "summary": "listMessages() and listMessagesLite() report paymentOutcome ('internalized', 'failed', 'skipped' or 'no-wallet-outputs') on each message that carried a payment, and keep that payment on the returned message unless the wallet accepted it (issue #503). Previously the payment envelope was always removed and a failed internalization was only logged, so acknowledging a listed message could delete the only copy of a payment. Both PeerMessage fields are optional and absent on messages without a payment. This release also carries the @bsv/sdk peer-floor raise from ^2.8.0 to ^2.8.6: SDK 2.8.0 through 2.8.5 derive the recipient key in Brc29RemittanceModule.acceptSettlement without forSelf, so PeerPayClient.acceptPayment() and rejectPayment() fail with brc29.internalize_failed for every valid incoming payment, and SDK 2.8.6 fixes acceptance. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "No migration required for the new fields: the PeerMessage paymentOutcome and retained payment are optional, and messages without a payment are unchanged. To avoid losing payments, store any returned payment (validate it first; it is the raw envelope value) before acknowledging its message. Install @bsv/sdk 2.8.6 or later alongside this package; payments refused on earlier SDKs remain pending in the message box and can be accepted after upgrading. No BRC100 wire or wallet-data change. SDK consumers may retain ^2.8.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Accepts canonical, bounded base64 transaction strings at PeerPay message receive boundaries alongside portable number arrays and legacy numeric-key JSON byte objects. Applies one validation path to list, live and indexed acceptance and rejects malformed or oversized encodings before wallet work or acknowledgement. listMessages() and listMessagesLite() report paymentOutcome ('internalized', 'failed', 'skipped' or 'no-wallet-outputs') on each message that carried a payment, and keep that payment on the returned message unless the wallet accepted it (issue #503). Previously the payment envelope was always removed and a failed internalization was only logged, so acknowledging a listed message could delete the only copy of a payment. Both PeerMessage fields are optional and absent on messages without a payment. This release also carries the @bsv/sdk peer-floor raise from ^2.8.0 to ^2.8.6: SDK 2.8.0 through 2.8.5 derive the recipient key in Brc29RemittanceModule.acceptSettlement without forSelf, so PeerPayClient.acceptPayment() and rejectPayment() fail with brc29.internalize_failed for every valid incoming payment, and SDK 2.8.6 fixes acceptance. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "Existing sender byte-array encoding and receiver byte forms remain compatible. Base64 must use the canonical standard alphabet and padding and decode to no more than 64 MiB. This receiver extension does not raise relay/intermediary limits or provide a durable refund/result journal; #548 and #503 retain those remaining milestones. Adopt the integrated SDK 3.0.0 candidate to obtain the independently tested BRC-29 recipient-key correction. No migration required for the new fields: the PeerMessage paymentOutcome and retained payment are optional, and messages without a payment are unchanged. To avoid losing payments, store any returned payment (validate it first; it is the raw envelope value) before acknowledging its message. Install @bsv/sdk 2.8.6 or later alongside this package; payments refused on earlier SDKs remain pending in the message box and can be accepted after upgrading. No BRC100 wire or wallet-data change. SDK consumers may retain ^2.8.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/overlay", @@ -126,15 +126,15 @@ "name": "@bsv/overlay-discovery-services", "publishedVersion": "2.2.6", "releaseType": "patch", - "summary": "Adds the unique MongoDB _id tie breaker to discovery pagination so bounded overflow probes preserve stable ordering. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/overlay-express", "publishedVersion": "2.7.3", "releaseType": "patch", - "summary": "Pages built-in discovery overflow probes within the discovery service 1000-row query bound, retaining the configured engine ceiling and explicit overflow failure. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "No wire, API, or database migration. Deploy the reviewed patch through the protected release workflow; retain existing validation and configured resource bounds. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.4.0 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/overlay-topics", @@ -153,16 +153,16 @@ { "name": "@bsv/payment-express-middleware", "publishedVersion": "2.1.8", - "releaseType": "patch", - "summary": "Removes middleware payment-header size rejection so received payments are validated regardless of header size. Retains the deprecated maxPaymentHeaderBytes option as an ignored source-compatibility field. Atomic BEEF framing, canonical base64, derivation verification, pricing, wallet acceptance and atomic replay protection remain. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "No BRC100 call, wire or wallet-data migration. maxPaymentHeaderBytes is now ignored, including existing explicit values: move transport policy to the HTTP server, CDN, proxy or WAF and remove the deprecated option. Validate the complete HTTP route for supported payment proofs. Published 2.1.8 is verified against protected release 36052862859 and its immutable source artifacts. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "releaseType": "minor", + "summary": "Adds receiver-first BRC-118 negotiation and bounded authenticated multipart payment extraction. Restores the original application payload and media type while retaining the existing BRC-105 amount, derivation, Atomic BEEF, internalization and replay checks. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "Requires @bsv/sdk ^2.9.0 || ^3.0.0. Set enableMultipart:true behind @bsv/auth-express-middleware captureRawBody:true installed before parsers. Support is advertised only on verified raw-byte requests. Configure maxPaymentBytes and maxPaymentBodyBytes with matching proxy/auth limits, and share a durable replay store across processes. Preserve recovery context after any submitted payment: the released ERR_PAYMENT_FAILED 400 does not prove rollback, and replay-store 503 is also ambiguous. Header-only deployments remain supported. See the BRC-118 guide. The retained SDK2 alternative starts at ^2.9.0; the integrated source graph now uses SDK3, including the paired BRC-118 correction. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/sdk", "publishedVersion": "2.8.10", "releaseType": "major", - "summary": "WalletClient and the binary BRC-100 wire no longer reject every acquireCertificate issuance: result validation required the returned fields to equal the request's, but issuance sends plaintext values and the certificate comes back with each value encrypted. An issuance result is now bound to exactly the requested field names; a direct acquisition is still bound to its requested encrypted values exactly. Removes the obsolete serial-DID token validation module and deep import. Native identity/certificate, wallet, transaction and authentication behavior is retained. Repairs cold BasePoint/JacobianPoint leaf exports with original-class initialization wrappers and exact declarations. The ESM UMD facade retains the established global effect and canonical class identities; the standalone classic bundle is preserved.", - "migration": "SDK3 is a separate breaking retirement candidate: remove identity/DIDTokenValidation imports and use @bsv/did BRC202 identity-key resolution plus optional @bsv/did/brc52 original-signature envelopes. Retain historical data and recover authenticated identity bindings; do not relabel serial DIDs. Existing acquisition-result validation fixes remain. First-party SDK3 peer alternatives require complete packed qualification and owner coordination; preserved SDK2 ranges are not silently narrowed. See docs/guides/identity-did-vc-migration.md." + "summary": "Adds BRC-118 negotiated, byte-preserving multipart payments to AuthFetch with bounded header/body budgets, prepare-before-broadcast validation, one-transaction submission/retry, cancellation and typed permanent recovery outcomes. Preserves nonempty non-multipart signature preimages and fixes BRC-29 recipient child-key derivation. Adds independent Python wire/preimage vectors and composed authenticated HTTP/proxy regression coverage. Preserves the configured BRC100 caller originator during automatic wallet discovery, so the HTTP WalletWire probe satisfies the existing transport caller binding. Binds the default HTTP JSON fetch receiver for browsers while retaining custom HTTP clients. Keeps transport preference, bounded discovery, operation timeout behavior, validation and origin checks unchanged. Restores signed listActions net amounts using their historical int64 wire encoding, matching existing JSON validation, while retaining canonical unsigned counts/lengths and nonnegative individual output values. Preserves native fetch body serialization and inferred Content-Type on ordinary HTTP fallback, including cached peers. WalletClient and the binary BRC-100 wire no longer reject every acquireCertificate issuance: result validation required the returned fields to equal the request's, but issuance sends plaintext values and the certificate comes back with each value encrypted. An issuance result is now bound to exactly the requested field names; a direct acquisition is still bound to its requested encrypted values exactly. Removes the obsolete serial-DID token validation module and deep import. Native identity/certificate, wallet and transaction APIs remain; the BRC-118 authentication changes above are included in the major candidate. Repairs cold BasePoint/JacobianPoint leaf exports with original-class initialization wrappers and exact declarations. The ESM UMD facade retains the established global effect and canonical class identities; the standalone classic bundle is preserved.", + "migration": "Use a BRC-100 wallet that honors noSend and sendWith and implements abortAction for automatic payments. Deploy raw-byte auth and multipart payment receivers first. Header-only or GET/HEAD requests whose payments require multipart fail before broadcast; select an application-supported body-bearing route explicitly. Serialize file-bearing FormData to owned bytes and retain its exact Content-Type; text-only FormData retains URL encoding. Reconcile submitted/uncertain payments before creating another spend, including after changed requirements, cancellation or proxy 413/431. Existing small header payments and nonempty non-multipart signing retain their wire format. Empty byte bodies use the BRC-104 -1 sentinel; clients and auth receivers that signed/reconstructed length 0 must adopt SDK 2.9.0 and auth middleware 2.3.0 together for those requests. BRC-29 receivers now derive their own child key. See the BRC-118 guide for limits and known external-peer differences. This candidate is not published and does not change current wallet-release pins. The included 2.8.3 discovery/browser-fetch and signed action-history repairs preserve the existing BRC100 API, historical wire bytes and account data. Older compatible clients retain their calls; applications bundling the affected 2.8.x binary client need the signed-history repair as well as the wallet. No ecosystem-wide migration is required for those compatibility fixes. SDK 2.8.3 was published by the protected SDK-only release workflow with verified registry integrity and provenance. The included 2.8.11 certificate-issuance compatibility repair requires no API, wire or wallet-data migration. SDK3 is a separate breaking retirement candidate: remove identity/DIDTokenValidation imports and use @bsv/did BRC202 identity-key resolution plus optional @bsv/did/brc52 original-signature envelopes. Retain historical data and recover authenticated identity bindings; do not relabel serial DIDs. Existing acquisition-result validation fixes remain. First-party SDK3 peer alternatives require complete packed qualification and owner coordination; preserved SDK2 ranges are not silently narrowed. See docs/guides/identity-did-vc-migration.md." }, { "name": "@bsv/simple", @@ -196,8 +196,8 @@ "name": "@bsv/wallet-helper", "publishedVersion": "0.1.8", "releaseType": "patch", - "summary": "Corrects caller-owned bilateral address derivation and hardens address generation, transaction building, prevout binding, P2PKH, Ordinal, OrdLock, preimage, sighash, OP_RETURN, script, amount, output, and final-wallet-transaction validation. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", - "migration": "getAddress now derives with forSelf: true and returns the caller-owned side of the bilateral relationship. Applications that stored or coordinated the previous peer-owned result must regenerate and exchange the corrected address before sending value. Amount must be an integer from 1 through 1,000 and counterparties must be valid public keys. Valid canonical transaction-builder flows remain supported; malformed, ambiguous, or wallet-mutated results now fail closed. Distributors must retain THIRD_PARTY_NOTICES.md and LICENSES/ with the package. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." + "summary": "Refreshes the packed first-party dependency ranges for the next wallet sync, proof recovery and BRC-118 release graph. This package adds no independent API or wire-format change. Adds an SDK3 peer alternative while preserving the existing SDK2 lower bound; clean packed SDK2.8.11 and SDK3 candidate consumers are documented in the identity/DID/VC migration guide.", + "migration": "No API migration. This unpublished candidate resolves the next source graph; adopt only after its dependencies are published. Existing wallet releases retain their current published pins. SDK consumers may retain ^2.1.6 or install the proposed SDK3 candidate. The immutable SDK2.8.11 reference has nine pre-existing SDK cold-leaf/UMD import defects; all historical floor versions are not qualified by this matrix. SDK3 removes the obsolete DID token API; use the identity-key DID and optional BRC52 adapters described in docs/guides/identity-did-vc-migration.md. No other public API or wire migration is introduced by the peer alternative." }, { "name": "@bsv/wallet-relay", @@ -209,23 +209,23 @@ { "name": "@bsv/wallet-toolbox", "publishedVersion": "2.14.4", - "releaseType": "patch", - "summary": "WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor. Public Argon2id declarations reference the typed hash-wasm package while retaining the same generic binary/string contract and emitted runtime bytes.", - "migration": "No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records." + "releaseType": "minor", + "summary": "Adds internal provider-owned journal floor/collection maintenance with complete generation validation, configured migration-owner exclusion, bounded cancellation/result lifetime and physical cleanup admission fencing. Adds internal monotonic continuity-floor transactions and primary-key tombstone collection bounded by examined records, with live-receipt pins and native WAL/RC/RR partial-delete/commit recovery proof. Adds bounded exact source-prefix receipts and explicit immutable receipt policy to owned SQLite/MySQL journal generations, with nonwaiting current reads, bounded expiry collection and process-loss recovery checks. Adds indexed exact SQLite journal source probes and an explicit durable bootstrap row allowance with atomic cursor/charge recovery and source-preserving exhaustion. Adds internal unadvertised SQL journal primitives and source-bound crash-resumable generation ownership, retaining bounded exact-revision pages and bootstrap. Accepts valid terminal all-zero checkpoint resets on unchanged backups while preserving state binding, timestamp monotonicity and nonterminal/partial-reset validation. Adds capability-gated resumable local sync with atomic page/checkpoint commits, cancellation, progress timings, fair foreground scheduling, provider/generation fencing and separated fixed/marginal page-cost fitting. Preserves returned/serialized sync failures and checks chain compatibility before writes. Repairs stale selected/input/broadcast proofs against canonical headers and paths, commits proof corrections with compare-and-set, and fences concurrent monitor repairs. Optimizes unfiltered IndexedDB label offsets without a schema change. Reduces redundant storage promise forwarding while retaining authorization inside exclusive ownership and preserving rejection and queue-release behavior. Shares the remote forwarding rejection boundary and avoids uncontended priority searches and repeated queue/checkpoint helper allocations without changing custom-transport rejection, wire, result or fairness contracts. Shares reader/writer admission helpers without moving authorization across the queue boundary, and uses a Map of property descriptors for in-place entity normalization without an intermediate mapping array. Date, null, byte, record-identity and non-enumerable own-field contracts are retained. Includes the 2.14.4 BRC-100 compatibility fixes: bounded certifier descriptions, requested discovery pagination, signed no-send change outpoints, originator-bound no-send aborts, and receiver-free browser fetch calls in chaintracker clients. WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Runs sequential storage and proof operations through a lazy asynchronous producer while retaining the eight-worker proof bound, failure draining, deterministic transaction order and Promise rejection contracts. Operations are pulled on demand rather than queued eagerly. Captures portable BRC-38 source metadata and all table reads in one provider read view and normalizes absent legacy JSON object fields without mutating source records or dropping array entries. Adds local retained SQLite/MySQL read views with bounded lifetime, single-read admission and physical cleanup ownership; these are excluded from RPC. Adds the version-one local wallet read-snapshot contract; SQL providers deliver profile-bound keyset pages with packed binary columns and payload-size preflight before fetching rows. Integrates supported ordinary local SQL push/pull/backup with a dedicated source reader and atomic durable destination sessions, normalized ID mappings and primary-epoch fencing. Adds syncToWriterResumable and exported local capability/checkpoint types; browser/mobile type availability does not add retained database or remote adapters. Push and backup preserve stored source primary metadata while pull retains the destination selection, including serialized fallback with an older manager cache. Adds internal shared SQL snapshot staging with immutable completed pages, exact retry receipts, profile isolation, bounded logical reservations and cleanup recovery. Adds a local SQL capture controller that binds original metadata/schema to one retained view, checks profile relationships, stores all thirteen raw tables in bounded binary frames and cleans up cancelled or failed captures. Adds bounded metadata-only receipt directories and source/profile/root validation for arbitrary-table page reads without fetching preceding payloads. Exact binding preimages and complete thirteen-table inclusion are preserved. Adds an internal durable creation-request lifecycle with deadline-bound request IDs, shared capacity admission, atomic archive assignment/ready publication and bounded terminal receipt retention. Integrates its capture controller with the provider-owned reader slot, reserves before pool acquisition, drains cancellation/shutdown and preserves ready archives across controller replacement. Physical cleanup failures retain reservations and fence admission, including source-opening failures. Adds a negotiated authenticated snapshot archive transport with prompt durable admission, exact profile-bound status/directory/page/cancel methods, client receipt/root validation and a dedicated two-MiB pre-parse response limit. Expiry reaping precedes admission; resource-limit terminal receipts remain distinct. HTTP shutdown awaits physical capture cleanup. This low-level transport does not yet integrate remote rows with ordinary sync/export or complete #544. Adds an unadvertised immutable remote row reader with v2 server-issued offers, require-existing admission, bounded same-request recovery, fixed leases and packed table paging. Source-only client adapters integrate with local sync through optional provider capabilities; archive positions are detached, checked and atomically persisted. Failure receipts remain observable until cancellation, and typed cancellation preserves unrelated errors. Server reader advertisement stays disabled pending cross-replica owner/physical-cleanup qualification. Preserves the public hash-wasm Argon2id generic contract in emitted declarations without exposing the untyped internal bundle. Adds a durable exact-claim source-owner fence: cross-controller cancellation and both request/archive cleanup retain quota until awaited source cleanup is acknowledged, and page append checks the request fence atomically. Adds fixed-slot backend guards and exact-owner recovery after physical connection closure on local SQLite WAL and same-server MySQL. Old unguarded owners remain reserved and changed backend identities fail closed; reader advertisement stays disabled pending complete qualification. Validates exact request-bound pending-cleanup receipts and polls cancellation with one fixed 30-second cleanup allowance, at most 32 attempts and bounded backoff, without renewing the source lease or hiding independent failures. Aborted I/O settles before cleanup returns. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor. Public Argon2id declarations reference the typed hash-wasm package while retaining the same generic binary/string contract and emitted runtime bytes. Keeps the provider source slot fenced after unproved native cleanup, even after its read promise settles; ordinary read failures remain retryable after proved cleanup. Adds auxiliary profile-key indexes for eight direct SQL tables without changing standard indexes or legacy OFFSET ordering; source triggers and resumable bootstrap keep membership coherent, and retained readers select the complete migration in their existing view. Adds auxiliary numeric relation keys for label/tag mappings with independent parent ownership bases, resumable composite bootstrap, retained-view selection and bounded MySQL primary-index page seeks before and after optimizer statistics refresh. Standard indexes, legacy OFFSET order and cursor/archive encodings remain unchanged. Adds resumable certificate-field indexes preserving source collation, exact text and independent direct/parent ownership, adopted by ordinary and archive readers inside their retained view. Adds resumable global proof/request reference indexes with exact counts, proof-presence guards and retained ordinary/archive selection; standard rows/indexes and BRC-38/cursor bytes are preserved. Adds conflict-safe SQLite auxiliary generations with metadata-bound displaced-identity witnesses, twelve bounded resumable bootstrap streams and bounded retirement of obsolete auxiliary rows. Reader adoption is pinned to journaled completion; MySQL maintenance and standard source records remain unchanged. Legacy SQLite retirement ownership binds object type and trigger name, refusing foreign views that share a trigger name before auxiliary data is retired. SQLite generation publication checks the configured migration-journal schema explicitly, preserving attached-schema journals and rejecting unrelated same-name publication records. Reconciles main\u2019s Postgres storage support while retaining its standard schema and legacy sync path; the current snapshot auxiliary migrations and capabilities remain limited to SQLite/MySQL. Adds internal owned journal capture: reserve two native pools before a nonwaiting writer fence, pin complete generation/profile/schema, commit the exact receipt and verify closure after releasing the barrier. Admission remains held through physical cleanup; an unproved close fences further sources. Preserves the original source/receipt failure alongside every rollback failure when owned journal transactions cannot prove cleanup; admission remains fenced until physical cleanup. Preserves queued primary reselection by checking no-op selections under existing ownership. A pending B switch followed by an A selection finishes with A, including after the earlier failure. A progress formatter failure before transition does not invalidate an in-flight sync. Invalidates cached primary authorization and reloads persisted selection after partial propagation failure; conflicting stores refuse active authorization, and recovery reload failures release queued ownership for retry. Allows exact proof and sync-checkpoint reads in provider-owned MySQL read-only snapshots without requesting write locks, while preserving row locking in ordinary writable transactions. Snapshot classification follows the transaction token across providers and is removed when its callback settles. Adds a complete critical read-snapshot consistency mutation target and generated direct/retained, cross-provider and independent-writer schedules, preserving ordinary unlocked lookups and checkpoint date/boolean normalization without changing the existing source API or isolation policy.", + "migration": "Unpublished journal generation helpers now require explicit receipt capacity/lifetime policy and refuse mismatched or earlier unregistered layouts. No registered migration or reader capability is added; capture publication, continuity-floor collection and receiver integration remain unfinished. Internal journal bootstrap callers must supply a fixed row allowance; this unpublished auxiliary layout rejects earlier unregistered generations rather than silently resetting their counters. Registered migration, reset and full resource-retention policy remain unfinished. The internal journal foundation adds no registered migration or public reader advertisement; full quota/capture/floor/receiver integration and full544 acceptance remain incomplete. The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. Use syncFromReaderResumable for the new local atomic-page path; legacy/custom or remote destinations retain exclusive fallback. Cancellation drains in-flight work and resumes from the durable destination checkpoint. Custom providers opt into concurrent reads/atomic pages only when those contracts hold; safe proof persistence additionally requires compareAndSetProvenTxProof. Configured proof services must validate active 80-byte headers. The original legacy page/proof subset needs no migration. The legacy timestamp/offset path may reread equal-time rows and does not provide a coherent source snapshot or streaming archive. Choose maxRoughSize deliberately for device/proxy memory budgets. Current wallet releases remain on their published dependency pins. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Retained views occupy a pool connection and require awaited close/closed cleanup; configure driver deadlines separately. IndexedDB retention remains unsupported. The retained-view primitive introduces no schema or wire migration. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. Apply 2026-09-30-002 add snapshot archive staging through the normal SQL migrate() entry point. Its capacity, manifest and page tables are auxiliary and excluded from BRC-38; no standard-table or legacy-checkpoint change is made. Current staging ceilings are eight handles, 128 MiB reserved globally, 32 MiB per capture, 1 MiB per page, 4096 pages and one hour lifetime. Reservations remain occupied through physical page cleanup. These preliminary limits do not qualify larger-wallet remote export; canonical portable semantic validation and operator resource policy are still required. The local capture controller requires a dedicated reader and an independent staging pool, with schema migrations outside active capture windows; it exposes no RPC or canonical BRC-38 output. Existing sync/backup behavior and browser/mobile adapters are unchanged by this staging component. The internal receipt directory requires no additional schema migration and exposes no RPC capability. Its one-MiB metadata budget does not replace a transport envelope limit; inclusion integrity is separate from portable semantic/proof validation. The existing unpublished 2.15 minor candidate covers these packed core bytes; no package is published by this change. Apply the additive 2026-09-30-003 add snapshot archive requests migration through migrate(); its auxiliary receipts stay outside portable wallet data. Pending requests use the existing archive capacity counters. Close outstanding requests before removing the request schema. The internal controller caps source lifetime at five minutes or the request remaining deadline and keeps the archive deadline fixed. Its per-process reader bound is distinct from shared logical staging quotas; driver cleanup deadlines remain an operator concern. The authenticated archive transport below consumes these internal components; remote row/manager integration and contention performance qualification remain incomplete. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. Apply 2026-10-01-001 add snapshot archive source owners before service capture. Stop and drain captures before downgrading; the migration refuses removal while owners remain and older draft binaries must not capture concurrently. Old unguarded owners retain ownership after unproved process loss. Standard tables and BRC-38/39 bytes are unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Apply 2026-10-01-002 add snapshot archive source guards through migrate(). Preserve the eight persistent slot bindings and SQLite sidecar guard files; never replace a bound file to free quota. New captures require better-sqlite3 WAL or the same actual MySQL server/database. Missing or changed backend identity refuses recovery. Drain all captures before downgrade; old unguarded owners require explicit source cleanup. Distributed filesystems, database relocation/restoration, PXC/load-balanced failover and global physical-connection limits require separate qualification. Reader advertisement stays disabled pending complete lifecycle qualification. After a source-cleanup error, do not reuse the affected provider for snapshot capture; retain its owner reservation until backend recovery proves physical closure. This correction adds no schema or wire change. Apply 2026-10-01-003 add snapshot profile key indexes through migrate(). This explicit nontransactional migration commits bounded 256-row auxiliary key/progress batches and journals only completion; preserve partial objects for retry, verify a terminated migrator before recovering any stale Knex lock, and drain readers before down. Standard rows/indexes and BRC-38/cursor bytes remain unchanged. Relationship/global-table indexing, commit-order high-water and nonblocking IndexedDB remain incomplete. Apply 2026-10-01-004 add snapshot relation key indexes through migrate(). Install/removal observers and producers preserve both ownership bases, including inconsistent mappings for closure refusal. Preserve partial auxiliary DDL/progress for retry, exclude other migrators before lock recovery, and drain readers before down. This migration remains outside BRC-38. Certificate-field/proof indexing, committed-change ordering, streaming and staged import remain incomplete; reader advertisement remains disabled. Apply 2026-10-01-005 add snapshot certificate field key indexes through migrate(). MySQL source and auxiliary tables require InnoDB and matching varchar(100) character set/collation; SQLite verifies the existing complete unique key and built-in BINARY/NOCASE/RTRIM order. Preserve partial owned DDL and committed bootstrap positions, exclude other migrators before verified lock recovery, and drain retained readers before down. Standard tables, indexes, legacy OFFSET and BRC-38/cursor bytes remain unchanged; proof/request indexing and the full program remain incomplete. Apply 2026-10-01-006 add snapshot global reference indexes through migrate(). Its reference edges, exact counts, presence guards and progress are auxiliary and excluded from BRC-38. Preserve partial DDL/committed positions; prove a stopped migrator and exclude others before lock recovery. MySQL requires standard unsigned/text metadata and InnoDB with explicit RESTRICT/NO ACTION mutations; implicit CASCADE/SET NULL actions are refused before DDL. Drain readers before down. The complete journal and progress are adopted within each pinned ordinary/archive view; source rows/indexes, legacy OFFSET and wire bytes stay unchanged. Commit ordering, tombstones, nonblocking IDB, remote destinations, streaming/staged portability and full acceptance remain incomplete; reader advertisement remains off. Apply 2026-10-02-001 repair snapshot SQLite conflict maintenance through migrate(). Preserve partial owned generation/progress for recovery and prove the stopped migrator before stale lock recovery. New readers use source queries while pending and adopt v2 inside the same retained view only after journal publication. Do not downgrade older snapshot binaries against invalidated legacy progress. Ordinary down refuses without deleting source rows; use a forward migration. Explicit dropAllData still deletes the complete wallet, including unpublished generation state. Freed SQLite pages may remain allocated for reuse; bounded retirement does not promise file shrink or full resource-budget acceptance. The SQLite migration-journal lookup correction needs no schema or wire migration; retain the existing tableName/schemaName binding. Postgres records these auxiliary snapshot migrations as no-ops and advertises no retained or paged source. Future Postgres snapshot support needs new forward migrations rather than rewriting the recorded entries. Internal openSnapshotJournalSource requires an existing file-backed better-sqlite3 WAL database or static mysql2 connection and an already complete explicitly owned journal generation. It registers no migration or incremental capability. A killed read view must be captured anew; retained prefix receipts do not reopen it. Runtime quotas, registered journal lifecycle, delta paging and receiver integration remain pending. Internal maintainSnapshotJournal owns generation validation and configured Knex migration exclusion before each bounded floor/collection transaction. Raw operator DDL remains outside that exclusion contract. Cancellation keeps shared capture admission until rollback or committed-result recovery and physical pool cleanup; unproved cleanup fences retained admission and remains observable through destruction. Runtime quotas, registered journal lifecycle/recovery and incremental reader advertisement remain unfinished; no new registered migration or public wire format is introduced. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation." }, { "name": "@bsv/wallet-toolbox-client", "publishedVersion": "2.14.4", - "releaseType": "patch", - "summary": "WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions.", - "migration": "No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records." + "releaseType": "minor", + "summary": "Accepts valid terminal all-zero checkpoint resets on unchanged backups while preserving state binding, timestamp monotonicity and nonterminal/partial-reset validation. Carries the compatible Toolbox resumable sync, fair storage scheduling, canonical proof recovery and IndexedDB paging improvements into the browser entry point. Adds native Chromium large-copy/foreground-latency acceptance alongside the packed browser contract. Reduces redundant storage promise forwarding while retaining authorization inside exclusive ownership and preserving rejection and queue-release behavior. Shares the remote forwarding rejection boundary and avoids uncontended priority searches and repeated queue/checkpoint helper allocations without changing custom-transport rejection, wire, result or fairness contracts. Shares reader/writer admission helpers without moving authorization across the queue boundary, and uses a Map of property descriptors for in-place entity normalization without an intermediate mapping array. Date, null, byte, record-identity and non-enumerable own-field contracts are retained. Includes the 2.14.4 BRC-100 compatibility fixes: bounded certifier descriptions, requested discovery pagination, signed no-send change outpoints, originator-bound no-send aborts, and receiver-free browser fetch calls in chaintracker clients. WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Captures portable BRC-38 source metadata and all table reads in one provider read view and normalizes absent legacy JSON object fields without mutating source records or dropping array entries. Exposes retained-view option/result types and the unsupported base-provider capability without adding a retained IndexedDB or remote implementation. Adds the version-one local wallet read-snapshot contract; SQL providers deliver profile-bound keyset pages with packed binary columns and payload-size preflight before fetching rows. Integrates supported ordinary local SQL push/pull/backup with a dedicated source reader and atomic durable destination sessions, normalized ID mappings and primary-epoch fencing. Adds syncToWriterResumable and exported local capability/checkpoint types; browser/mobile type availability does not add retained database or remote adapters. Push and backup preserve stored source primary metadata while pull retains the destination selection, including serialized fallback with an older manager cache. Adds a negotiated authenticated snapshot archive transport with prompt durable admission, exact profile-bound status/directory/page/cancel methods, client receipt/root validation and a dedicated two-MiB pre-parse response limit. Expiry reaping precedes admission; resource-limit terminal receipts remain distinct. HTTP shutdown awaits physical capture cleanup. This low-level transport does not yet integrate remote rows with ordinary sync/export or complete #544. Adds an unadvertised immutable remote row reader with v2 server-issued offers, require-existing admission, bounded same-request recovery, fixed leases and packed table paging. Source-only client adapters integrate with local sync through optional provider capabilities; archive positions are detached, checked and atomically persisted. Failure receipts remain observable until cancellation, and typed cancellation preserves unrelated errors. Server reader advertisement stays disabled pending cross-replica owner/physical-cleanup qualification. Preserves the public hash-wasm Argon2id generic contract in emitted declarations without exposing the untyped internal bundle. Validates exact request-bound pending-cleanup receipts and polls cancellation with one fixed 30-second cleanup allowance, at most 32 attempts and bounded backoff, without renewing the source lease or hiding independent failures. Aborted I/O settles before cleanup returns. Preserves queued primary reselection by checking no-op selections under existing ownership. A pending B switch followed by an A selection finishes with A, including after the earlier failure. A progress formatter failure before transition does not invalidate an in-flight sync. Invalidates cached primary authorization and reloads persisted selection after partial propagation failure; conflicting stores refuse active authorization, and recovery reload failures release queued ownership for retry.", + "migration": "The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. Use the additive syncFromReaderResumable API and its progress/cancellation contracts; custom providers keep the exclusive fallback until they implement the capability contract. No IndexedDB schema migration is required. Source timestamps remain inclusive rather than a coherent snapshot; equal-time boundary rows can be reread. Select explicit page bytes for constrained devices. Publication and downstream wallet adoption are separate actions. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Do not assume retained-view support from the types; inspect the concrete local provider capability. Existing scoped capture remains unchanged. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Backend-bound orphan recovery remains required before reader advertisement. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation." }, { "name": "@bsv/wallet-toolbox-mobile", "publishedVersion": "2.14.4", - "releaseType": "patch", - "summary": "WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor.", - "migration": "No public API, wire, database, or consumer migration is required. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records." + "releaseType": "minor", + "summary": "Accepts valid terminal all-zero checkpoint resets on unchanged backups while preserving state binding, timestamp monotonicity and nonterminal/partial-reset validation. Carries the compatible Toolbox sync contracts, fair scheduling and canonical proof recovery into the mobile entry point without adding Node or IndexedDB dependencies. Reduces redundant storage promise forwarding while retaining authorization inside exclusive ownership and preserving rejection and queue-release behavior. Shares the remote forwarding rejection boundary and avoids uncontended priority searches and repeated queue/checkpoint helper allocations without changing custom-transport rejection, wire, result or fairness contracts. Shares reader/writer admission helpers without moving authorization across the queue boundary, and uses a Map of property descriptors for in-place entity normalization without an intermediate mapping array. Date, null, byte, record-identity and non-enumerable own-field contracts are retained. Includes the 2.14.4 BRC-100 compatibility fixes: bounded certifier descriptions, requested discovery pagination, signed no-send change outpoints, originator-bound no-send aborts, and receiver-free browser fetch calls in chaintracker clients. WalletPermissionsManager retires signed no-send ownership and reference aliases after createAction or signAction reports sending or unproven, including sendWith-only calls and successful members of an undelayed review error. Failed or unreported actions retain their originator binding for retry or abort. Cleanup visits only the retired transaction's references and preserves aliases reused by newer actions. Captures portable BRC-38 source metadata and all table reads in one provider read view and normalizes absent legacy JSON object fields without mutating source records or dropping array entries. Exposes retained-view option/result types and the unsupported base-provider capability without adding a retained IndexedDB or remote implementation. Adds the version-one local wallet read-snapshot contract; SQL providers deliver profile-bound keyset pages with packed binary columns and payload-size preflight before fetching rows. Integrates supported ordinary local SQL push/pull/backup with a dedicated source reader and atomic durable destination sessions, normalized ID mappings and primary-epoch fencing. Adds syncToWriterResumable and exported local capability/checkpoint types; browser/mobile type availability does not add retained database or remote adapters. Push and backup preserve stored source primary metadata while pull retains the destination selection, including serialized fallback with an older manager cache. Adds a negotiated authenticated snapshot archive transport with prompt durable admission, exact profile-bound status/directory/page/cancel methods, client receipt/root validation and a dedicated two-MiB pre-parse response limit. Expiry reaping precedes admission; resource-limit terminal receipts remain distinct. HTTP shutdown awaits physical capture cleanup. This low-level transport does not yet integrate remote rows with ordinary sync/export or complete #544. Adds an unadvertised immutable remote row reader with v2 server-issued offers, require-existing admission, bounded same-request recovery, fixed leases and packed table paging. Source-only client adapters integrate with local sync through optional provider capabilities; archive positions are detached, checked and atomically persisted. Failure receipts remain observable until cancellation, and typed cancellation preserves unrelated errors. Server reader advertisement stays disabled pending cross-replica owner/physical-cleanup qualification. Preserves the public hash-wasm Argon2id generic contract in emitted declarations without exposing the untyped internal bundle. Validates exact request-bound pending-cleanup receipts and polls cancellation with one fixed 30-second cleanup allowance, at most 32 attempts and bounded backoff, without renewing the source lease or hiding independent failures. Aborted I/O settles before cleanup returns. Adds an SDK3 peer alternative while retaining the existing SDK2.8.0 floor. Preserves queued primary reselection by checking no-op selections under existing ownership. A pending B switch followed by an A selection finishes with A, including after the earlier failure. A progress formatter failure before transition does not invalidate an in-flight sync. Invalidates cached primary authorization and reloads persisted selection after partial propagation failure; conflicting stores refuse active authorization, and recovery reload failures release queued ownership for retry.", + "migration": "The unchanged-backup checkpoint correction requires no API, wire or schema migration. Lost terminal acknowledgements resume from the durable destination checkpoint. The additive syncFromReaderResumable API uses atomic local pages only for capable local providers; remote/custom destinations keep the serialized fallback. Existing mobile runtime/crypto backend registration and RPC wires remain unchanged. The original legacy page subset needs no migration. Reconcile cancellation and lost acknowledgements using the durable checkpoint; do not infer coherent source snapshots. Current wallet releases are not required to consume this candidate. The included 2.14.4 compatibility fixes retain stored trust settings and require no additional wire or database migration. The included legacy compatibility fixes need no public API or wire migration. Pending no-send actions and retryable failures retain the same originator-bound abort behavior; positively queued or broadcast actions no longer retain session ownership records. Custom StorageProvider implementations opt in with supportsReadSnapshot/readSnapshot and must honor the view token on every query. Pass requireSnapshot:true to exportBRC38/exportBRC38Json or exportBRC39 options to refuse unsupported coherent capture before table reads; old custom-provider calls retain their documented caller-quiesced legacy path. Built-in SQLite/MySQL and IndexedDB implement the local read view. MySQL configures the next transaction on one reserved connection without changing session defaults and closes failed connections before pool release. IndexedDB writers wait during capture; document/file materialization and the existing remote-client limitation remain. This intermediate checkpoint does not complete #544 or change BRC-38/39 wire bytes. Do not assume retained-view support from the types; inspect the concrete local provider capability. Existing scoped capture remains unchanged. openWalletReadSnapshot is an additive local SQL capability, excluded from RPC and unsupported on IndexedDB. Its cursor lives only as long as the retained view; restart after expiry or process loss. Page limits bound rows and conservative stored-payload charges, not wire bytes or RSS; oversized individual rows explicitly refuse pending large-value streaming. The page primitive preserves standard-table indexes and legacy OFFSET/checkpoint behavior. For the new local SQL sync path apply 2026-09-30-001 add durable snapshot sync through migrate(): three auxiliary tables plus a primary-change trigger preserve legacy sync-map JSON. SQLite requires file-backed WAL; MySQL requires a static database connection. Explicit row/reference/retention limits and unsupported configurations use serialized fallback. snapshotSync:false on current binaries is the supported forward rollback with schema retained; stop sessions before any binary downgrade, whose older migration code may reject newer journal entries. Source loss restarts traversal under a new view with durable mappings retained. Primary reconciliation, remote/IDB retained support, streaming and staged restore remain required; #544 is incomplete. No BRC-38/39 bytes change. The additive getSnapshotArchiveTransport(identityKey) client API requires an authenticated compatible advertisement; old or disabled peers decline it. Server capability requires the existing archive/request migrations, static WAL/MySQL support, snapshotSync enabled and sufficient envelopes. Set snapshotArchives:false on either server or client for rollback with schema retained. Keep the exact creation tuple/deadline for lost-response recovery; never substitute a new source after active work begins. Existing ordinary RPC, BRC-38/39 bytes and public CORS behavior are preserved. The additive snapshotArchiveReaderVersion:1 setting is separate from legacy snapshotArchive capability; current servers do not yet advertise it. Existing clients and old servers keep their established path. Cursor archivePosition metadata is optional and legacy cursor JSON is unchanged. Ordinary remote source use requires completed lifecycle qualification; no distributed physical-drain or new remote destination guarantee is provided. The Argon2id declaration correction requires no migration or runtime change; SDK peer ranges remain unchanged. The unadvertised v2 reader path distinguishes pending cleanup from the existing true acknowledgement. Use matching candidate clients to wait for that acknowledgement; an older strict reader rejects the new pending receipt rather than claiming completion. Legacy cancellation retains its existing success/error response shapes; it does not use the new v2 pending receipt. Backend-bound orphan recovery remains required before reader advertisement. The queued setActive correction requires no API, wire or schema migration. Primary reconciliation remains serialized pending the complete #544 implementation." }, { "name": "create-bsv-app", diff --git a/governance/repository-health/baselines.json b/governance/repository-health/baselines.json index 982206c58..8ed4db1bd 100644 --- a/governance/repository-health/baselines.json +++ b/governance/repository-health/baselines.json @@ -14,11 +14,11 @@ "run": "https://github.com/BSV-blockchain/ts-stack/actions/runs/30144812565" }, "conformance": { - "passed": 6489, + "passed": 6495, "skipped": 211, - "total": 6700, - "vectorFiles": 77, - "run": "Local: pnpm --filter @bsv/conformance-runner-ts test (2026-09-24); 6489 vector cases plus 2 metadata/wire tests, 211 unchanged governed skips; adds required empty historical BRC100 display metadata interoperability vector" + "total": 6706, + "vectorFiles": 78, + "run": "Local: pnpm --filter @bsv/conformance-runner-ts test (2026-09-28); 6495 vector cases plus 2 metadata/wire tests passed, 211 unchanged governed skips; retains current-main BRC100 display metadata and six independent Python-verified BRC-118 vectors." }, "testExceptions": { "explicitSkipDeclarations": 60, @@ -313,8 +313,8 @@ "@bsv/paymail": "2.4.10", "@bsv/402-pay": "0.3.3", "@bsv/auth": "0.1.6", - "@bsv/auth-express-middleware": "2.2.9", - "@bsv/payment-express-middleware": "2.1.9", + "@bsv/auth-express-middleware": "2.3.0", + "@bsv/payment-express-middleware": "2.2.0", "@bsv/teranode-listener": "1.1.7", "@bsv/gasp": "1.3.8", "@bsv/overlay": "2.6.3", @@ -327,8 +327,8 @@ "@bsv/btms-permission-module": "1.2.2", "@bsv/ecpm-permission-module": "0.1.2", "@bsv/wallet-relay": "0.5.2", - "@bsv/wallet-toolbox-client": "2.14.5", - "@bsv/wallet-toolbox-mobile": "2.14.5", - "@bsv/wallet-toolbox": "2.14.5" + "@bsv/wallet-toolbox-client": "2.15.0", + "@bsv/wallet-toolbox-mobile": "2.15.0", + "@bsv/wallet-toolbox": "2.15.0" } } diff --git a/governance/repository-health/exceptions.json b/governance/repository-health/exceptions.json index 5eb6b920f..9b212dcce 100644 --- a/governance/repository-health/exceptions.json +++ b/governance/repository-health/exceptions.json @@ -73,9 +73,9 @@ { "id": "brace-expansion-jest-override", "category": "override", - "target": "pnpm-workspace.yaml brace-expansion@<5.0.11 and four standalone Jest service manifests pinning brace-expansion 5.0.12", + "target": "pnpm-workspace.yaml and four standalone Jest service manifests overriding brace-expansion <5.0.12", "owner": "ts-stack-maintainers", - "reason": "Jest 30.4.2 and Stryker still constrain reporting and coverage paths to minimatch releases that admit older brace-expansion ranges. GHSA-mh99-v99m-4gvg required 5.0.8, GHSA-rgw5-rvv9-x895 required 5.0.9, GHSA-6j4f-fj2g-mc7p requires 5.0.10, and GHSA-qhr7-859c-m2p7 requires 5.0.11. GHSA-q2hr-2g5m-vwhr affects every release below 5.0.12, so the four standalone Jest service manifests pin 5.0.12. The workspace substitution remains 5.0.11.", + "reason": "Jest 30.4.2 and Stryker still constrain reporting and coverage paths to minimatch releases that require older brace-expansion ranges. Review on 2026-09-30 advances the existing substitution from 5.0.9 to 5.0.12 for GHSA-qhr7-859c-m2p7, GHSA-6j4f-fj2g-mc7p and GHSA-q2hr-2g5m-vwhr. The patched release preserves the CommonJS/ESM exports, types and Node engine range; frozen installs, service tests and coverage/mutation qualification verify the existing compatibility contract.", "evidence": [ "pnpm-workspace.yaml#overrides", "pnpm-lock.yaml#overrides", @@ -86,10 +86,11 @@ "infra/wab/package.json#overrides", "https://github.com/advisories/GHSA-mh99-v99m-4gvg", "https://github.com/advisories/GHSA-rgw5-rvv9-x895", - "https://github.com/advisories/GHSA-6j4f-fj2g-mc7p", + "https://github.com/bsv-blockchain/ts-stack/issues/324", "https://github.com/advisories/GHSA-qhr7-859c-m2p7", + "https://github.com/advisories/GHSA-6j4f-fj2g-mc7p", "https://github.com/advisories/GHSA-q2hr-2g5m-vwhr", - "https://github.com/bsv-blockchain/ts-stack/issues/324" + "https://github.com/juliangruber/brace-expansion/compare/v5.0.9...v5.0.12" ], "created": "2026-07-25", "reviewBy": "2026-10-31", @@ -246,7 +247,7 @@ "category": "override", "target": "tools/codegen/node/package.json overrides for @redocly/openapi-core@1.34.17", "owner": "ts-stack-maintainers", - "reason": "The latest openapi-typescript release still depends on Redocly 1.x ranges that resolve vulnerable js-yaml and minimatch paths, so the isolated codegen lock substitutes compatible patched releases and verifies the generated output byte-for-byte.", + "reason": "The latest openapi-typescript release still depends on Redocly 1.x ranges that resolve vulnerable js-yaml and minimatch paths, so the isolated codegen lock substitutes compatible patched releases and verifies the generated output byte-for-byte. Review on 2026-09-30 advances only the existing nested brace-expansion substitution to 5.0.12 for the three follow-up advisories, preserving exports, types and the Node engine range.", "evidence": [ "tools/codegen/node/package.json#overrides", "tools/codegen/node/package-lock.json", @@ -255,10 +256,14 @@ "https://github.com/advisories/GHSA-mh99-v99m-4gvg", "https://github.com/advisories/GHSA-rgw5-rvv9-x895", "https://github.com/openapi-ts/openapi-typescript/blob/openapi-typescript%407.13.0/packages/openapi-typescript/package.json", - "https://github.com/advisories/GHSA-2883-xcg3-v3hh" + "https://github.com/advisories/GHSA-2883-xcg3-v3hh", + "https://github.com/advisories/GHSA-qhr7-859c-m2p7", + "https://github.com/advisories/GHSA-6j4f-fj2g-mc7p", + "https://github.com/advisories/GHSA-q2hr-2g5m-vwhr", + "https://github.com/juliangruber/brace-expansion/compare/v5.0.9...v5.0.12" ], "created": "2026-07-25", - "reviewBy": "2026-09-25", + "reviewBy": "2026-10-30", "removeWhen": "Remove when a supported openapi-typescript release natively resolves non-vulnerable js-yaml, minimatch, and brace-expansion versions, then regenerate and compare every committed output." }, { @@ -409,6 +414,44 @@ "created": "2026-10-02", "reviewBy": "2026-11-02", "removeWhen": "Remove when a released @stryker-mutator/vitest-runner supports Vitest 5 test-name chains, the governed Stryker pin is moved to it, and fund-wallet-cli and payment-402 meet their policy floors on Vitest 5 with zero survived mutants reporting testsCompleted 0." + }, + { + "id": "metro-file-map-picomatch-advisory-repair", + "category": "override", + "target": "pnpm-workspace.yaml exact metro-file-map@0.87.1>micromatch removal, Picomatch2.3.2 package extension and paired watcher patch", + "owner": "ts-stack-maintainers", + "reason": "No patched braces release or compatible Metro/file-map/micromatch upgrade is available for high GHSA-vfj7-8cjw-p6xm. Metro-file-map0.87.1 only consumes micromatch.some(), which already delegates to Picomatch2.3.2. The source/distribution patch retains that exact loop and adds the same exact direct matcher dependency, permitting scoped removal of micromatch and its unused vulnerable braces closure. All other package versions/importers and installation controls are preserved. This is a temporary dependency repair, not an advisory exclusion.", + "evidence": [ + "https://github.com/advisories/GHSA-vfj7-8cjw-p6xm", + "https://github.com/micromatch/braces/pull/72", + "patches/metro-file-map@0.87.1.patch", + "scripts/lib/metro-watcher-contract.mjs", + "scripts/lib/fixtures/metro-watcher-contract.json", + "scripts/check-wallet-toolbox-platform.mjs", + "https://github.com/bsv-blockchain/ts-stack/discussions/757" + ], + "created": "2026-10-02", + "reviewBy": "2026-11-01", + "removeWhen": "Remove the patch, exact-version package extension and scoped removal together after an official compatible Metro/file-map release removes the affected dependency path; require the complete watcher oracle, fresh unexcluded audit, frozen graph and packed Metro/Hermes platform checks before retirement." + }, + { + "id": "standalone-nodemon-chokidar-advisory-repair", + "category": "override", + "target": "Nodemon-parent chokidar4.0.3 in infra/uhrp-server-basic, infra/uhrp-server-cloud-bucket and infra/wab", + "owner": "ts-stack-maintainers", + "reason": "No compatible released Nodemon update removes high GHSA-vfj7-8cjw-p6xm from its Chokidar3/braces graph. Nodemon already expands watch globs into literal directories. A source-owned adapter restores the former anymatch3 ignore predicate, including literal-directory recursion, custom cwd, regex/functions and dotfile behavior, while retaining the actual CLI, manual restart, Node/ts-node preloads and shutdown. WAB replaces ts-node-dev with the same tested watcher instead of forcing its unsupported glob API through Chokidar4. Only each Nodemon parent receives the substitution; production service startup and storage/HTTP contracts are unchanged. This is a temporary dependency repair, not an advisory exclusion.", + "evidence": [ + "https://github.com/advisories/GHSA-vfj7-8cjw-p6xm", + "https://github.com/paulmillr/chokidar/releases/tag/4.0.3", + "https://github.com/remy/nodemon/blob/v3.1.14/bin/nodemon.js", + "infra/uhrp-server-basic/dev/bin/nodemon.js", + "infra/uhrp-server-basic/dev/check-watch.cjs", + "governance/service-runtime-copy-policy.json", + "https://github.com/bsv-blockchain/ts-stack/discussions/757" + ], + "created": "2026-10-03", + "reviewBy": "2026-11-03", + "removeWhen": "Remove all three scoped substitutions and the compatibility adapter together after an official compatible Nodemon release removes the affected dependency closure and preserves the full ignore/restart/preload/manual-restart/shutdown contract; require fresh frozen unexcluded audits, complete service builds/tests and protected Linux image qualification before retirement." } ] } diff --git a/governance/service-runtime-copy-policy.json b/governance/service-runtime-copy-policy.json index c5fedf61e..142d362aa 100644 --- a/governance/service-runtime-copy-policy.json +++ b/governance/service-runtime-copy-policy.json @@ -1,8 +1,8 @@ { "schemaVersion": 1, - "lastReviewed": "2026-09-27", + "lastReviewed": "2026-10-03", "owner": "ts-stack-maintainers", - "rationale": "Standalone image build contexts retain a small number of runtime sources that are canonically owned elsewhere. These copies are synchronized byte-for-byte so published packages and official images cannot drift.", + "rationale": "Standalone image build contexts retain a small number of runtime sources that are canonically owned elsewhere. These copies are synchronized byte-for-byte so published packages and official images cannot drift. The same rule owns standalone development watcher adapters and their native regression tests.", "copies": [ { "canonicalSource": "infra/uhrp-server-basic/src/resourceLimits.ts", @@ -112,6 +112,20 @@ { "canonicalSource": "infra/uhrp-server-basic/src/chirp/bodyMiddleware.ts", "synchronizedSources": ["infra/uhrp-server-cloud-bucket/src/chirp/bodyMiddleware.ts"] + }, + { + "canonicalSource": "infra/uhrp-server-basic/dev/bin/nodemon.js", + "synchronizedSources": [ + "infra/uhrp-server-cloud-bucket/dev/bin/nodemon.js", + "infra/wab/dev/bin/nodemon.js" + ] + }, + { + "canonicalSource": "infra/uhrp-server-basic/dev/check-watch.cjs", + "synchronizedSources": [ + "infra/uhrp-server-cloud-bucket/dev/check-watch.cjs", + "infra/wab/dev/check-watch.cjs" + ] } ] } diff --git a/governance/test-quality/policy.json b/governance/test-quality/policy.json index 68b4dc4d3..9cc9e67ed 100644 --- a/governance/test-quality/policy.json +++ b/governance/test-quality/policy.json @@ -518,6 +518,152 @@ ], "owner": "ts-stack-maintainers", "reviewBy": "2026-10-04" + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet retained database-view admission, expiry, cancellation and physical cleanup ownership, profile-bound keyset cursors and bounded packed payload admission", + "target": "Randomized read, cancellation, close and expiry schedules with an independent admission and cleanup model; randomized source profiles, payload/page limits and independent writes against pinned SQL views", + "invariants": [ + "At most one database read is admitted and concurrent or nested reads do not queue.", + "Closing, cancellation and expiry reject later admission and discard late successful results.", + "Capacity remains occupied until physical reads and transaction cleanup settle.", + "Every completed lifecycle removes its expiry timer and rejects further reads.", + "Pinned keyset traversal returns every original owned row and tombstone exactly once under independent writes, within both page limits; cursor retries repeat the same page.", + "Auxiliary profile membership equals committed source ownership through generated insert, update, move, delete, rollback, repeated migration and rebuild schedules.", + "Numeric relationship membership equals the OR of current parent owners through generated map/parent rekeys, profile moves, tombstones, physical deletions, rollback and repeated migration; each ownership basis is retained independently." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournal.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet exact journal revisions, bounded metadata pages and bootstrap, full-table observers, source binding and owned crash-resumable generation lifecycle", + "target": "Generated exact signed63 revision and metadata cursor schedules, plus thirteen-table insert/update/rekey/replacement/delete/rollback histories checked against independent source ownership oracles", + "invariants": [ + "Canonical decimal revisions compare exactly across the signed63 domain, including values above Number precision.", + "Bounded metadata pages preserve composite byte ordering, fixed upper bounds and examined-row limits without skipping tied revisions.", + "Generated source changes retain exact physical generations and ownership across all thirteen standard tables, including replacement, rollback and observer order variations.", + "Invalid source/schema or ownership evidence never silently adopts foreign DDL, and interrupted installation resumes only the persisted epoch and exact authorized next object." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet durable profile-bound snapshot merges, normalized ID mappings, acknowledged cursors and primary-generation fencing", + "target": "Randomized bounded SQL page, stale preparation, acknowledgement loss and source-view replacement schedules with an independent profile/content oracle", + "invariants": [ + "Only the acknowledged destination cursor advances after each atomic page commit.", + "Concurrent prepared pages cannot overwrite a newer checkpoint, and replacing a source view fences its old pages.", + "Incoming IDs and parent references are mapped within one wallet profile without duplicating logical records.", + "A restart explicitly begins the replacement view at its first table while preserving existing destination data and legacy checkpoints." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncDestination.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet durable snapshot destination binding, atomic checkpoint admission and primary-generation fencing", + "target": "Randomized original profile metadata, same-view binding drift, independent profiles and replacement views", + "invariants": [ + "Same-view metadata changes reject without modifying the durable checkpoint.", + "A replacement view starts at table zero with a fresh session while preserving another profile." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet bounded snapshot row detachment and profile/source-scoped normalized parent and child ID mapping", + "target": "Randomized overlapping incoming IDs across profiles and sources with independent expected mapping and persisted-row oracles", + "invariants": [ + "Incoming parent IDs resolve only within the selected profile and source storage.", + "Only newly learned child IDs are persisted, existing parent and foreign mappings remain unchanged, and missing parents reject." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet coherent SQL snapshot capture and staging, profile closure, immutable receipts, quotas and cleanup accounting", + "target": "Persistent bounded archive staging across capture retries, incomplete captures, independent readers and expired/cancelled cleanup", + "invariants": [ + "A partial or expired capture is never readable as a completed archive.", + "Exact page retries preserve immutable bytes and the independent hash-chain receipt without double charging.", + "The authenticated profile cannot read or release another profile's archive.", + "Capacity remains reserved through interrupted cleanup and is released exactly once after every page is deleted.", + "A complete SQL capture contains the selected profile’s original rows; cancellation removes all staged pages and reservations without activating wallet data." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet authenticated snapshot receipt directories, original binding bytes, complete table hash chains and page inclusion", + "target": "Generated complete table ranges, independent receipt hashes and changed-byte rejection", + "invariants": [ + "Every bounded receipt chain preserves all thirteen table positions and row totals.", + "Arbitrary table pages match their previously verified receipt before decoding.", + "Changed payload, receipt or previously pinned root bytes reject.", + "Caller mutation does not alter verified receipt positions or metadata." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet remote snapshot creation request identity, durable receipt publication, bounded reservation, exact source ownership, backend guards and physical cleanup recovery", + "target": "Generated SQL claim, retry, capture publication, cross-controller cancellation, delayed native close, guarded recovery and successor ownership schedules", + "invariants": [ + "A retry returns the original request and immutable ready archive without a second admission or writer token.", + "Pending requests reserve the same shared capacity used by local archives before source acquisition.", + "Archive assignment and ready receipt publication commit atomically.", + "Remote cancellation, repeated reaping and incorrect acknowledgements retain pages and quota while the exact source owner remains.", + "Terminal requests retain bounded receipts and release their logical reservation exactly once after the source owner acknowledges physical cleanup.", + "Backend guards retain quota through native close; recovery fences the exact terminal owner and old acknowledgements cannot release a successor." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet authenticated snapshot HTTP negotiation, exact protocol fields, profile binding, immutable client receipts and bounded transport lifecycle", + "target": "Generated deadline-bound requests, exact RPC shapes and client receipt validation", + "invariants": [ + "Requests and receipts retain the original immutable identity and deadline across start/status retries.", + "Every RPC accepts only its exact typed fields and never an internal ownership token.", + "Ready receipts bind one archive/root while every distinct terminal state remains explicit.", + "Unexpected identity, expiry or ownership fields are rejected rather than silently normalized." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet immutable remote snapshot row decoding, fixed archive cursors, bounded leases, exact admission recovery and pending cleanup", + "target": "Generated packed frame and page boundaries, immutable retry positions and per-page allocation accounting", + "invariants": [ + "Every generated row is returned exactly once in original source order across arbitrary frame and caller-page boundaries.", + "The archive position binds the exact frame and last consumed row without reading preceding tables.", + "Same-position retries reproduce identical rows, cursor and allocation charge.", + "A cursor for another archive is rejected and each opened reader cancels its original immutable request exactly once." + ] + }, + { + "path": "packages/wallet/wallet-toolbox/src/storage/portable/mysqlReadSnapshot.property.test.ts", + "manifest": "packages/wallet/wallet-toolbox/package.json", + "risk": "critical", + "boundary": "Wallet callback-owned coherent transaction classification, exact proof/checkpoint consistent reads and preservation of writable authority locks", + "target": "Generated direct/retained read schedules across providers with interleaved independent writable transactions, partial/exact selectors and callback failures", + "invariants": [ + "Provider-owned coherent snapshots never request writable row locks.", + "Independent ordinary transactions preserve each existing exact-selector lock requirement.", + "Snapshot classification follows the transaction token across provider instances and never marks an independent transaction.", + "Direct and retained callbacks preserve exact failure causes and release their database connection.", + "Every generated schedule preserves the complete selector/query count without an unbounded queue." + ] } ], "exclusions": [ diff --git a/governance/third-party-materials.json b/governance/third-party-materials.json index 840285017..041b2c957 100644 --- a/governance/third-party-materials.json +++ b/governance/third-party-materials.json @@ -26,7 +26,7 @@ { "id": "docs-site-bundled-materials", "path": "governance/docs-site-bundled-materials.json", - "sha256": "afe1bacd45b0bda408a1db3b19e1783805071938023d384a7ad1b071a0808e4a" + "sha256": "5f6c8ca03121e0b588582b2f5efa734e539534d1dc1ce4f8a83ffad8ad7192c1" }, { "id": "license-continuity", diff --git a/infra/chaintracks-server/package-lock.json b/infra/chaintracks-server/package-lock.json index 7f5291b7e..2d7e162ba 100644 --- a/infra/chaintracks-server/package-lock.json +++ b/infra/chaintracks-server/package-lock.json @@ -3400,9 +3400,9 @@ } }, "node_modules/ip-address": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.3.1.tgz", - "integrity": "sha512-1e9d3kb97NHJTIJDZW9rKqW2h6+dFa50Dy0fpPSMQp2ADje5gvKsXmdiK6dwY5t76TaTt5+P5N1Y/LoToIxP6g==", + "version": "10.7.2", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.2.tgz", + "integrity": "sha512-7H/2gFSIitxc0hG3nOI1glS8QLo/EHBFFLk8vEUjXY/xu0AdL8jZ9U1IzO2PUm0d2D/ofQcAifb0g6OBkt8U7w==", "license": "MIT", "engines": { "node": ">= 12" diff --git a/infra/message-box-server/README.md b/infra/message-box-server/README.md index bcdf6ad42..f67ecfcb7 100644 --- a/infra/message-box-server/README.md +++ b/infra/message-box-server/README.md @@ -9,6 +9,15 @@ The maintained source lives in [`bsv-blockchain/ts-stack`](https://github.com/bsv-blockchain/ts-stack/tree/main/infra/message-box-server). The service is distributed as a container; it is not a public npm package. +The standalone lock selects `@fastify/busboy` 3.2.2 within its existing +transitive dependency range, addressing +[GHSA-xjh9-v7x6-24jw](https://github.com/advisories/GHSA-xjh9-v7x6-24jw) and +[GHSA-x8mw-p69m-v3mx](https://github.com/advisories/GHSA-x8mw-p69m-v3mx). +No service API, request ceiling, runtime requirement or database migration +changes. Use the frozen lock and the protected container release process; +existing deployed images receive the fix only through a separately authorized +image promotion. + See [Service Resource Profiles](../../docs/reference/service-resource-profiles.md) for all runtime ceilings, BRC-105 pricing, capacity evidence, and HPA prerequisites. diff --git a/infra/message-box-server/package-lock.json b/infra/message-box-server/package-lock.json index 30756e884..a4169da27 100644 --- a/infra/message-box-server/package-lock.json +++ b/infra/message-box-server/package-lock.json @@ -1286,9 +1286,9 @@ } }, "node_modules/@fastify/busboy": { - "version": "3.2.0", - "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.2.0.tgz", - "integrity": "sha512-m9FVDXU3GT2ITSe0UaMA5rU3QkfC/UXtCU8y0gSN/GugTqtVldOBWIB5V6V3sbmenVZUIpU6f+mPEO2+m5iTaA==", + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/@fastify/busboy/-/busboy-3.2.2.tgz", + "integrity": "sha512-yXSS27qPExaXeuLvMRMXOLtpipzfQYNjG3FkunDWKGfMYjKuhFXko9CVzqxm8jcF+lmtS9Fd89QNdh9XDjnbNg==", "license": "MIT" }, "node_modules/@firebase/app-check-interop-types": { @@ -7968,9 +7968,9 @@ } }, "node_modules/ip-address": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", - "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", + "version": "10.7.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.1.tgz", + "integrity": "sha512-4OUAqU9Z1i3vCnS05hzGiFnEMDpQ+62pAD/MVQOp83fYyNC8GleCqaS0QikQBmcWCrKFiUs/B8ztRRiYOAXuCA==", "license": "MIT", "engines": { "node": ">= 12" diff --git a/infra/uhrp-server-basic/README.md b/infra/uhrp-server-basic/README.md index e7f933d79..db904022e 100644 --- a/infra/uhrp-server-basic/README.md +++ b/infra/uhrp-server-basic/README.md @@ -112,3 +112,23 @@ recreated the same pathname. Heartbeats keep live locks current, but an orphan left by a crashed process requires operator removal after every writer using the shared `CHIRP_DATA_DIR` has been stopped. Never delete a lock while any CHIRP replica may still be writing. + +## Development watcher + +`npm run dev` uses the locked Nodemon CLI with a source-owned Chokidar 4 +compatibility adapter. It watches TypeScript, existing JavaScript/JSON extensions, +new source files and `.env`, retains the Node/ts-node telemetry preload, and +supports manual `rs` restarts. Existing ignored glob, directory, regex and +function options retain Chokidar 3 matching behavior. WAB replaces ts-node-dev +with this same CLI; production startup, HTTP contracts and persisted data are +unchanged. No public npm package version or consumer migration is required for +these standalone service development tools. + +The parent-scoped Chokidar substitution removes the affected braces dependency +without an advisory exclusion. The dated dependency registry owns its removal +condition. `npm test` first runs the actual locked watcher regression, including +clean shutdown, using the actual service development recipe. WAB also retains +compiler-configuration restarts. The basic UHRP service owns the adapter and +regression; the root service-copy generator synchronizes cloud UHRP and WAB. +Protected Linux image and exact-head CI checks must qualify release candidates; +source changes do not update deployed images. diff --git a/infra/uhrp-server-basic/dev/bin/nodemon.js b/infra/uhrp-server-basic/dev/bin/nodemon.js new file mode 100644 index 000000000..fe89254b7 --- /dev/null +++ b/infra/uhrp-server-basic/dev/bin/nodemon.js @@ -0,0 +1,48 @@ +'use strict' + +const path = require('node:path') +const anymatch = require('anymatch') +const isGlob = require('is-glob') +const { bus } = require('nodemon/lib/utils') +const { rulesToMonitor } = require('nodemon/lib/monitor/match') + +function unix(value) { + const slashes = value.replaceAll('\\', '/') + return (slashes.startsWith('//') ? '/' : '') + slashes.replaceAll(/\/{2,}/g, '/') +} +function normalizeIgnored(value, cwd) { + if (typeof value !== 'string') return value + const joined = path.isAbsolute(value) ? value : path.join(cwd ?? '', value) + return unix(path.normalize(unix(joined))) +} + +/** Nodemon already expands watched globs into literal directories. Chokidar4 + * also needs the former glob-aware ignore predicate, including literal-directory + * recursion and custom ignored functions/regexes. Retain its published anymatch + * matcher instead of the removed braces-dependent watch-path expansion. */ +function applyOptions(config) { + const options = config.options.watchOptions ?? {} + let ignored + if (Object.hasOwn(options, 'ignored')) ignored = options.ignored + else { + ignored = rulesToMonitor([], Array.from(config.options.ignore), config).map(pattern => + pattern.slice(1) + ) + const dotFile = /[/\\]\./ + if (!config.dirs.some(directory => dotFile.test(directory))) ignored.push(dotFile) + } + const normalized = (Array.isArray(ignored) ? ignored : [ignored]).map(value => + normalizeIgnored(value, options.cwd) + ) + const directories = normalized + .filter(value => typeof value === 'string' && !isGlob(value)) + .map(value => value + '/**') + const predicate = anymatch([...normalized, ...directories], undefined, { dot: true }) + config.options.watchOptions = { ...options, ignored: (file, stats) => predicate([file, stats]) } +} + +module.exports = applyOptions +if (require.main === module) { + bus.on('config:update', applyOptions) + require('nodemon/bin/nodemon.js') +} diff --git a/infra/uhrp-server-basic/dev/check-watch.cjs b/infra/uhrp-server-basic/dev/check-watch.cjs new file mode 100644 index 000000000..7e380ce0e --- /dev/null +++ b/infra/uhrp-server-basic/dev/check-watch.cjs @@ -0,0 +1,195 @@ +'use strict' + +const assert = require('node:assert/strict') +const fs = require('node:fs/promises') +const { readFileSync } = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { spawn } = require('node:child_process') +const { test } = require('node:test') +const applyOptions = require('./bin/nodemon.js') +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) + +test('development ignore options retain legacy glob, literal-directory, regex, function, cwd and dotfile behavior', () => { + const cases = [ + { ignored: '/repo/cache', file: '/repo/cache/child/file.ts', expected: true }, + { ignored: '/repo/cache', file: '/repo/cache-other/file.ts', expected: false }, + { ignored: 'cache', cwd: '/repo', file: '/repo/cache/file.ts', expected: true }, + { + ignored: String.raw`cache\nested`, + cwd: '/repo', + file: '/repo/cache/nested/file.ts', + expected: true + }, + { ignored: '**/generated/**', file: '/repo/generated/file.ts', expected: true }, + { ignored: '**/generated/**', file: '/repo/src/file.ts', expected: false }, + { ignored: ['**/*.ts', '!**/keep.ts'], file: '/repo/src/drop.ts', expected: true }, + { ignored: ['**/*.ts', '!**/keep.ts'], file: '/repo/src/keep.ts', expected: false }, + { ignored: /\.map$/, file: '/repo/file.map', expected: true }, + { ignored: /\.map$/, file: '/repo/file.ts', expected: false }, + { + ignored: (_file, stats) => stats?.marker === true, + file: '/repo/a.ts', + stats: { marker: true }, + expected: true + }, + { ignored: (_file, stats) => stats?.marker === true, file: '/repo/a.ts', expected: false }, + { ignored: undefined, file: '/repo/a.ts', expected: false }, + { ignored: [], file: '/repo/a.ts', expected: false } + ] + for (const item of cases) { + const config = { + dirs: ['/repo/src'], + options: { + ignore: [], + watchOptions: { ignored: item.ignored, cwd: item.cwd, usePolling: true, interval: 71 } + } + } + applyOptions(config) + assert.equal( + config.options.watchOptions.ignored(item.file, item.stats), + item.expected, + item.file + ) + assert.equal(config.options.watchOptions.usePolling, true) + assert.equal(config.options.watchOptions.interval, 71) + } + const defaults = { dirs: ['/repo/src'], options: { ignore: ['**/node_modules/**'] } } + applyOptions(defaults) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/.hidden.ts'), true) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/node_modules/pkg/a.ts'), true) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/a.ts'), false) + const explicitDotfile = { dirs: ['/repo/.env', '/repo/src'], options: { ignore: [] } } + applyOptions(explicitDotfile) + assert.equal(explicitDotfile.options.watchOptions.ignored('/repo/.env'), false) +}) + +test( + 'actual locked watcher restarts TS, env and new source, retains preloads/manual restart, ignores dependencies and stops', + { timeout: 60000 }, + async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts-stack-dev-watch-')) + let child, + exited, + output = '', + errors = '', + forced = false + const starts = () => [...output.matchAll(/READY:(\d+):(\d+)/g)] + async function waitFor(predicate, description, remaining = 400) { + if (remaining === 0) throw new Error(`Watcher did not ${description}: ${output}\n${errors}`) + if (predicate()) return + if (child.exitCode !== null || child.signalCode !== null) + throw new Error(`Watcher exited during ${description}: ${errors}`) + await delay(25) + return waitFor(predicate, description, remaining - 1) + } + try { + await fs.mkdir(path.join(directory, 'src/node_modules'), { recursive: true }) + await fs.mkdir(path.join(directory, 'src/generated'), { recursive: true }) + await fs.symlink( + path.resolve(__dirname, '../node_modules'), + path.join(directory, 'node_modules'), + 'junction' + ) + await fs.writeFile( + path.join(directory, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + module: 'commonjs', + target: 'es2022', + strict: true, + skipLibCheck: true, + types: ['node'] + } + }) + ) + await fs.writeFile(path.join(directory, '.env'), 'synthetic ordinary configuration') + await fs.writeFile(path.join(directory, 'src/dependency.ts'), 'export const value = 1') + await fs.writeFile( + path.join(directory, 'src/telemetry.ts'), + "console.log('PRELOAD:' + process.pid)" + ) + const recipe = JSON.parse(readFileSync(path.resolve(__dirname, '../package.json'), 'utf8')) + .scripts.dev + const prefix = 'node dev/bin/nodemon.js ' + assert.ok(recipe.startsWith(prefix) && recipe.endsWith('"')) + const [watch, execution] = recipe.slice(prefix.length, -1).split(' --exec "') + assert.ok(execution) + const options = watch.split(/\s+/) + const entry = execution.match(/src\/(index|server)\.ts$/)?.[0] + assert.ok(entry) + await fs.writeFile( + path.join(directory, entry), + "import { value } from './dependency'; console.log('READY:' + process.pid + ':' + value); setInterval(() => {}, 1000)" + ) + child = spawn( + process.execPath, + [ + path.join(__dirname, 'bin/nodemon.js'), + ...options, + '--ignore', + 'src/generated/**', + '--exec', + execution + ], + { + cwd: directory, + detached: true, + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, CI: '1', NO_UPDATE_NOTIFIER: '1' } + } + ) + exited = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + child.stdout.on('data', data => { + output += data.toString() + }) + child.stderr.on('data', data => { + errors += data.toString() + }) + await waitFor(() => starts().length === 1, 'start the synthetic TypeScript process') + await delay(500) + await fs.writeFile(path.join(directory, 'src/dependency.ts'), 'export const value = 2') + await waitFor(() => starts().length === 2, 'restart for a TypeScript dependency') + assert.equal(starts()[1][2], '2') + await delay(500) + await fs.writeFile(path.join(directory, 'src/node_modules/ignored.js'), 'module.exports = 1') + await fs.writeFile( + path.join(directory, 'src/generated/ignored.ts'), + 'export const ignored = 1' + ) + await delay(700) + assert.equal(starts().length, 2) + await fs.writeFile(path.join(directory, '.env'), 'synthetic changed configuration') + await waitFor(() => starts().length === 3, 'restart for the explicitly watched env file') + await delay(500) + await fs.writeFile(path.join(directory, 'src/new-file.ts'), 'export const newFile = 1') + await waitFor(() => starts().length === 4, 'restart for a new TypeScript source file') + child.stdin.write('rs\n') + await waitFor(() => starts().length === 5, 'accept a manual restart') + if (options.includes('tsconfig.json')) { + await delay(500) + await fs.appendFile(path.join(directory, 'tsconfig.json'), '\n') + await waitFor(() => starts().length === 6, 'restart for the watched compiler configuration') + } + assert.equal([...output.matchAll(/PRELOAD:/g)].length, starts().length) + } finally { + if (child?.exitCode === null && child.signalCode === null) { + process.kill(-child.pid, 'SIGTERM') + const escalation = setTimeout(() => { + forced = true + process.kill(-child.pid, 'SIGKILL') + }, 5000) + try { + await exited + } finally { + clearTimeout(escalation) + } + } + await fs.rm(directory, { recursive: true, force: true }) + } + assert.equal(forced, false, 'the CLI and owned child must exit without forced termination') + } +) diff --git a/infra/uhrp-server-basic/package-lock.json b/infra/uhrp-server-basic/package-lock.json index 4a48155a4..7084da903 100644 --- a/infra/uhrp-server-basic/package-lock.json +++ b/infra/uhrp-server-basic/package-lock.json @@ -40,6 +40,8 @@ "@types/node": "^26.6.1", "@types/prettyjson": "^0.0.33", "@typescript/native": "npm:typescript@7.0.2", + "anymatch": "^3.1.3", + "is-glob": "^4.0.3", "jest": "^30.5.1", "nodemon": "^3.1.14", "oxlint": "^1.83.0", @@ -4714,18 +4716,6 @@ "node": "*" } }, - "node_modules/binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/body-parser": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", @@ -4776,18 +4766,6 @@ "node": "20 || >=22" } }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/browserslist": { "version": "4.29.0", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.29.0.tgz", @@ -4937,27 +4915,19 @@ } }, "node_modules/chokidar": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", - "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", "dev": true, + "license": "MIT", "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" + "readdirp": "^4.0.1" }, "engines": { - "node": ">= 8.10.0" + "node": ">= 14.16.0" }, "funding": { "url": "https://paulmillr.com/funding/" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" } }, "node_modules/ci-info": { @@ -5547,18 +5517,6 @@ "node": "^12.20 || >= 14.13" } }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/finalhandler": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", @@ -5716,20 +5674,6 @@ "node": ">= 0.8" } }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -5883,18 +5827,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/google-logging-utils": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", @@ -6102,9 +6034,9 @@ "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" }, "node_modules/ip-address": { - "version": "10.3.1", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.3.1.tgz", - "integrity": "sha512-1e9d3kb97NHJTIJDZW9rKqW2h6+dFa50Dy0fpPSMQp2ADje5gvKsXmdiK6dwY5t76TaTt5+P5N1Y/LoToIxP6g==", + "version": "10.7.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.1.tgz", + "integrity": "sha512-4OUAqU9Z1i3vCnS05hzGiFnEMDpQ+62pAD/MVQOp83fYyNC8GleCqaS0QikQBmcWCrKFiUs/B8ztRRiYOAXuCA==", "license": "MIT", "engines": { "node": ">= 12" @@ -6125,18 +6057,6 @@ "dev": true, "license": "MIT" }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/is-extglob": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/is-extglob/-/is-extglob-2.1.1.tgz", @@ -6176,15 +6096,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -7949,15 +7860,17 @@ } }, "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, + "license": "MIT", "engines": { - "node": ">=8.10.0" + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" } }, "node_modules/real-require": { @@ -8550,18 +8463,6 @@ "integrity": "sha512-P4nbQYQfePJxRSmY+v/KINxVucm4NF3p3s7pJveMTtom52FR4YGltUQLB8idDXwDDWW+eYrWDFbuzUnjoWHF7g==", "license": "MIT" }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", diff --git a/infra/uhrp-server-basic/package.json b/infra/uhrp-server-basic/package.json index d0d3e4fb2..8aa9e6923 100644 --- a/infra/uhrp-server-basic/package.json +++ b/infra/uhrp-server-basic/package.json @@ -3,7 +3,10 @@ "version": "0.1.46", "overrides": { "brace-expansion": "5.0.12", - "gaxios": "7.3.0" + "gaxios": "7.3.0", + "nodemon": { + "chokidar": "4.0.3" + } }, "engines": { "node": ">=24 <25", @@ -58,11 +61,12 @@ "scripts": { "start": "ts-node -r ./src/telemetry.ts src/index.ts", "start:prod": "node --require ./out/src/telemetry.js out/src/index.js", - "dev": "nodemon --watch .env --watch src --exec \"node --inspect=0.0.0.0 -r ts-node/register -r ./src/telemetry.ts src/index.ts\"", + "dev": "node dev/bin/nodemon.js --ext ts,js,mjs,cjs,json,env --watch .env --watch src --exec \"node --inspect=0.0.0.0 -r ts-node/register -r ./src/telemetry.ts src/index.ts\"", "build": "tsc", "test": "jest", "test:watch": "jest --watch", - "lint": "oxlint src --deny-warnings" + "lint": "oxlint src dev --deny-warnings", + "pretest": "node --test dev/check-watch.cjs" }, "devDependencies": { "@types/body-parser": "^1.19.6", @@ -70,6 +74,8 @@ "@types/node": "^26.6.1", "@types/prettyjson": "^0.0.33", "@typescript/native": "npm:typescript@7.0.2", + "anymatch": "^3.1.3", + "is-glob": "^4.0.3", "jest": "^30.5.1", "nodemon": "^3.1.14", "oxlint": "^1.83.0", diff --git a/infra/uhrp-server-cloud-bucket/README.md b/infra/uhrp-server-cloud-bucket/README.md index 1ac1195f1..eeb948f87 100644 --- a/infra/uhrp-server-cloud-bucket/README.md +++ b/infra/uhrp-server-cloud-bucket/README.md @@ -409,3 +409,23 @@ route, and failure-path checks above pass. © 2025 – Feel free to adapt, improve, and PR! Advertisement, owner metadata and renewal operations use the same `GCP_STORAGE_CREDS`/`GCP_PROJECT_ID` identity as signed uploads. When credentials are explicitly configured, those operations must not fall back to the runtime metadata server. Unset credentials retain ADC for installations that intentionally use a runtime service account. Malformed configured credentials fail without logging their contents. + +## Development watcher + +`npm run dev` uses the locked Nodemon CLI with a source-owned Chokidar 4 +compatibility adapter. It watches TypeScript, existing JavaScript/JSON extensions, +new source files and `.env`, retains the Node/ts-node telemetry preload, and +supports manual `rs` restarts. Existing ignored glob, directory, regex and +function options retain Chokidar 3 matching behavior. WAB replaces ts-node-dev +with this same CLI; production startup, HTTP contracts and persisted data are +unchanged. No public npm package version or consumer migration is required for +these standalone service development tools. + +The parent-scoped Chokidar substitution removes the affected braces dependency +without an advisory exclusion. The dated dependency registry owns its removal +condition. `npm test` first runs the actual locked watcher regression, including +clean shutdown, using the actual service development recipe. WAB also retains +compiler-configuration restarts. The basic UHRP service owns the adapter and +regression; the root service-copy generator synchronizes cloud UHRP and WAB. +Protected Linux image and exact-head CI checks must qualify release candidates; +source changes do not update deployed images. diff --git a/infra/uhrp-server-cloud-bucket/dev/bin/nodemon.js b/infra/uhrp-server-cloud-bucket/dev/bin/nodemon.js new file mode 100644 index 000000000..fe89254b7 --- /dev/null +++ b/infra/uhrp-server-cloud-bucket/dev/bin/nodemon.js @@ -0,0 +1,48 @@ +'use strict' + +const path = require('node:path') +const anymatch = require('anymatch') +const isGlob = require('is-glob') +const { bus } = require('nodemon/lib/utils') +const { rulesToMonitor } = require('nodemon/lib/monitor/match') + +function unix(value) { + const slashes = value.replaceAll('\\', '/') + return (slashes.startsWith('//') ? '/' : '') + slashes.replaceAll(/\/{2,}/g, '/') +} +function normalizeIgnored(value, cwd) { + if (typeof value !== 'string') return value + const joined = path.isAbsolute(value) ? value : path.join(cwd ?? '', value) + return unix(path.normalize(unix(joined))) +} + +/** Nodemon already expands watched globs into literal directories. Chokidar4 + * also needs the former glob-aware ignore predicate, including literal-directory + * recursion and custom ignored functions/regexes. Retain its published anymatch + * matcher instead of the removed braces-dependent watch-path expansion. */ +function applyOptions(config) { + const options = config.options.watchOptions ?? {} + let ignored + if (Object.hasOwn(options, 'ignored')) ignored = options.ignored + else { + ignored = rulesToMonitor([], Array.from(config.options.ignore), config).map(pattern => + pattern.slice(1) + ) + const dotFile = /[/\\]\./ + if (!config.dirs.some(directory => dotFile.test(directory))) ignored.push(dotFile) + } + const normalized = (Array.isArray(ignored) ? ignored : [ignored]).map(value => + normalizeIgnored(value, options.cwd) + ) + const directories = normalized + .filter(value => typeof value === 'string' && !isGlob(value)) + .map(value => value + '/**') + const predicate = anymatch([...normalized, ...directories], undefined, { dot: true }) + config.options.watchOptions = { ...options, ignored: (file, stats) => predicate([file, stats]) } +} + +module.exports = applyOptions +if (require.main === module) { + bus.on('config:update', applyOptions) + require('nodemon/bin/nodemon.js') +} diff --git a/infra/uhrp-server-cloud-bucket/dev/check-watch.cjs b/infra/uhrp-server-cloud-bucket/dev/check-watch.cjs new file mode 100644 index 000000000..7e380ce0e --- /dev/null +++ b/infra/uhrp-server-cloud-bucket/dev/check-watch.cjs @@ -0,0 +1,195 @@ +'use strict' + +const assert = require('node:assert/strict') +const fs = require('node:fs/promises') +const { readFileSync } = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { spawn } = require('node:child_process') +const { test } = require('node:test') +const applyOptions = require('./bin/nodemon.js') +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) + +test('development ignore options retain legacy glob, literal-directory, regex, function, cwd and dotfile behavior', () => { + const cases = [ + { ignored: '/repo/cache', file: '/repo/cache/child/file.ts', expected: true }, + { ignored: '/repo/cache', file: '/repo/cache-other/file.ts', expected: false }, + { ignored: 'cache', cwd: '/repo', file: '/repo/cache/file.ts', expected: true }, + { + ignored: String.raw`cache\nested`, + cwd: '/repo', + file: '/repo/cache/nested/file.ts', + expected: true + }, + { ignored: '**/generated/**', file: '/repo/generated/file.ts', expected: true }, + { ignored: '**/generated/**', file: '/repo/src/file.ts', expected: false }, + { ignored: ['**/*.ts', '!**/keep.ts'], file: '/repo/src/drop.ts', expected: true }, + { ignored: ['**/*.ts', '!**/keep.ts'], file: '/repo/src/keep.ts', expected: false }, + { ignored: /\.map$/, file: '/repo/file.map', expected: true }, + { ignored: /\.map$/, file: '/repo/file.ts', expected: false }, + { + ignored: (_file, stats) => stats?.marker === true, + file: '/repo/a.ts', + stats: { marker: true }, + expected: true + }, + { ignored: (_file, stats) => stats?.marker === true, file: '/repo/a.ts', expected: false }, + { ignored: undefined, file: '/repo/a.ts', expected: false }, + { ignored: [], file: '/repo/a.ts', expected: false } + ] + for (const item of cases) { + const config = { + dirs: ['/repo/src'], + options: { + ignore: [], + watchOptions: { ignored: item.ignored, cwd: item.cwd, usePolling: true, interval: 71 } + } + } + applyOptions(config) + assert.equal( + config.options.watchOptions.ignored(item.file, item.stats), + item.expected, + item.file + ) + assert.equal(config.options.watchOptions.usePolling, true) + assert.equal(config.options.watchOptions.interval, 71) + } + const defaults = { dirs: ['/repo/src'], options: { ignore: ['**/node_modules/**'] } } + applyOptions(defaults) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/.hidden.ts'), true) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/node_modules/pkg/a.ts'), true) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/a.ts'), false) + const explicitDotfile = { dirs: ['/repo/.env', '/repo/src'], options: { ignore: [] } } + applyOptions(explicitDotfile) + assert.equal(explicitDotfile.options.watchOptions.ignored('/repo/.env'), false) +}) + +test( + 'actual locked watcher restarts TS, env and new source, retains preloads/manual restart, ignores dependencies and stops', + { timeout: 60000 }, + async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts-stack-dev-watch-')) + let child, + exited, + output = '', + errors = '', + forced = false + const starts = () => [...output.matchAll(/READY:(\d+):(\d+)/g)] + async function waitFor(predicate, description, remaining = 400) { + if (remaining === 0) throw new Error(`Watcher did not ${description}: ${output}\n${errors}`) + if (predicate()) return + if (child.exitCode !== null || child.signalCode !== null) + throw new Error(`Watcher exited during ${description}: ${errors}`) + await delay(25) + return waitFor(predicate, description, remaining - 1) + } + try { + await fs.mkdir(path.join(directory, 'src/node_modules'), { recursive: true }) + await fs.mkdir(path.join(directory, 'src/generated'), { recursive: true }) + await fs.symlink( + path.resolve(__dirname, '../node_modules'), + path.join(directory, 'node_modules'), + 'junction' + ) + await fs.writeFile( + path.join(directory, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + module: 'commonjs', + target: 'es2022', + strict: true, + skipLibCheck: true, + types: ['node'] + } + }) + ) + await fs.writeFile(path.join(directory, '.env'), 'synthetic ordinary configuration') + await fs.writeFile(path.join(directory, 'src/dependency.ts'), 'export const value = 1') + await fs.writeFile( + path.join(directory, 'src/telemetry.ts'), + "console.log('PRELOAD:' + process.pid)" + ) + const recipe = JSON.parse(readFileSync(path.resolve(__dirname, '../package.json'), 'utf8')) + .scripts.dev + const prefix = 'node dev/bin/nodemon.js ' + assert.ok(recipe.startsWith(prefix) && recipe.endsWith('"')) + const [watch, execution] = recipe.slice(prefix.length, -1).split(' --exec "') + assert.ok(execution) + const options = watch.split(/\s+/) + const entry = execution.match(/src\/(index|server)\.ts$/)?.[0] + assert.ok(entry) + await fs.writeFile( + path.join(directory, entry), + "import { value } from './dependency'; console.log('READY:' + process.pid + ':' + value); setInterval(() => {}, 1000)" + ) + child = spawn( + process.execPath, + [ + path.join(__dirname, 'bin/nodemon.js'), + ...options, + '--ignore', + 'src/generated/**', + '--exec', + execution + ], + { + cwd: directory, + detached: true, + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, CI: '1', NO_UPDATE_NOTIFIER: '1' } + } + ) + exited = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + child.stdout.on('data', data => { + output += data.toString() + }) + child.stderr.on('data', data => { + errors += data.toString() + }) + await waitFor(() => starts().length === 1, 'start the synthetic TypeScript process') + await delay(500) + await fs.writeFile(path.join(directory, 'src/dependency.ts'), 'export const value = 2') + await waitFor(() => starts().length === 2, 'restart for a TypeScript dependency') + assert.equal(starts()[1][2], '2') + await delay(500) + await fs.writeFile(path.join(directory, 'src/node_modules/ignored.js'), 'module.exports = 1') + await fs.writeFile( + path.join(directory, 'src/generated/ignored.ts'), + 'export const ignored = 1' + ) + await delay(700) + assert.equal(starts().length, 2) + await fs.writeFile(path.join(directory, '.env'), 'synthetic changed configuration') + await waitFor(() => starts().length === 3, 'restart for the explicitly watched env file') + await delay(500) + await fs.writeFile(path.join(directory, 'src/new-file.ts'), 'export const newFile = 1') + await waitFor(() => starts().length === 4, 'restart for a new TypeScript source file') + child.stdin.write('rs\n') + await waitFor(() => starts().length === 5, 'accept a manual restart') + if (options.includes('tsconfig.json')) { + await delay(500) + await fs.appendFile(path.join(directory, 'tsconfig.json'), '\n') + await waitFor(() => starts().length === 6, 'restart for the watched compiler configuration') + } + assert.equal([...output.matchAll(/PRELOAD:/g)].length, starts().length) + } finally { + if (child?.exitCode === null && child.signalCode === null) { + process.kill(-child.pid, 'SIGTERM') + const escalation = setTimeout(() => { + forced = true + process.kill(-child.pid, 'SIGKILL') + }, 5000) + try { + await exited + } finally { + clearTimeout(escalation) + } + } + await fs.rm(directory, { recursive: true, force: true }) + } + assert.equal(forced, false, 'the CLI and owned child must exit without forced termination') + } +) diff --git a/infra/uhrp-server-cloud-bucket/package-lock.json b/infra/uhrp-server-cloud-bucket/package-lock.json index 492db7830..ef8716687 100644 --- a/infra/uhrp-server-cloud-bucket/package-lock.json +++ b/infra/uhrp-server-cloud-bucket/package-lock.json @@ -47,7 +47,9 @@ "@types/node": "^26.6.1", "@types/prettyjson": "^0.0.33", "@typescript/native": "npm:typescript@7.0.2", + "anymatch": "^3.1.3", "ejs": "^6.0.1", + "is-glob": "^4.0.3", "jest": "^30.5.1", "nodemon": "^3.1.14", "oxlint": "^1.83.0", @@ -5147,19 +5149,6 @@ "node": "*" } }, - "node_modules/binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/body-parser": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.3.0.tgz", @@ -5233,19 +5222,6 @@ "node": "20 || >=22" } }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "license": "MIT", - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/browserslist": { "version": "4.28.7", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.7.tgz", @@ -5425,28 +5401,19 @@ } }, "node_modules/chokidar": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", - "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", "dev": true, "license": "MIT", "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" + "readdirp": "^4.0.1" }, "engines": { - "node": ">= 8.10.0" + "node": ">= 14.16.0" }, "funding": { "url": "https://paulmillr.com/funding/" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" } }, "node_modules/ci-info": { @@ -6238,19 +6205,6 @@ "node": "^12.20 || >= 14.13" } }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "license": "MIT", - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/finalhandler": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", @@ -6411,21 +6365,6 @@ "node": ">= 0.8" } }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "license": "MIT", - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -6574,19 +6513,6 @@ "url": "https://github.com/sponsors/isaacs" } }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "license": "ISC", - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/google-auth-library": { "version": "9.15.1", "resolved": "https://registry.npmjs.org/google-auth-library/-/google-auth-library-9.15.1.tgz", @@ -6913,9 +6839,9 @@ } }, "node_modules/ip-address": { - "version": "10.4.0", - "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.4.0.tgz", - "integrity": "sha512-oSK96Grm3aP6OrS263xVxbNDGVL7rzBtYdpGqlDG8iQdoenDoTs/nkki+DflYbAEE8Xl6o5YxhxlrKvI3nqKXQ==", + "version": "10.7.1", + "resolved": "https://registry.npmjs.org/ip-address/-/ip-address-10.7.1.tgz", + "integrity": "sha512-4OUAqU9Z1i3vCnS05hzGiFnEMDpQ+62pAD/MVQOp83fYyNC8GleCqaS0QikQBmcWCrKFiUs/B8ztRRiYOAXuCA==", "license": "MIT", "engines": { "node": ">= 12" @@ -6937,19 +6863,6 @@ "dev": true, "license": "MIT" }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "license": "MIT", - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/is-core-module": { "version": "2.16.2", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.2.tgz", @@ -7007,16 +6920,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -8962,16 +8865,17 @@ } }, "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", "dev": true, "license": "MIT", - "dependencies": { - "picomatch": "^2.2.1" - }, "engines": { - "node": ">=8.10.0" + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" } }, "node_modules/real-require": { @@ -9843,19 +9747,6 @@ "node": ">=8" } }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "license": "MIT", - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", diff --git a/infra/uhrp-server-cloud-bucket/package.json b/infra/uhrp-server-cloud-bucket/package.json index 4f1783ec8..3c0811ea8 100644 --- a/infra/uhrp-server-cloud-bucket/package.json +++ b/infra/uhrp-server-cloud-bucket/package.json @@ -4,7 +4,10 @@ "overrides": { "brace-expansion": "5.0.12", "gaxios": "7.3.0", - "uuid": "11.1.1" + "uuid": "11.1.1", + "nodemon": { + "chokidar": "4.0.3" + } }, "engines": { "node": ">=24 <25", @@ -64,13 +67,14 @@ "scripts": { "start": "ts-node -r ./src/telemetry.ts src/index.ts", "start:prod": "node --require ./out/src/telemetry.js --max-http-header-size=512000 out/src/index.js", - "dev": "nodemon --watch .env --watch src --exec \"node --inspect=0.0.0.0 -r ts-node/register -r ./src/telemetry.ts src/index.ts\"", + "dev": "node dev/bin/nodemon.js --ext ts,js,mjs,cjs,json,env --watch .env --watch src --exec \"node --inspect=0.0.0.0 -r ts-node/register -r ./src/telemetry.ts src/index.ts\"", "build": "tsc", "test": "jest", "test:watch": "jest --watch", - "lint": "oxlint src scripts --deny-warnings", + "lint": "oxlint src scripts dev --deny-warnings", "secrets:staging": "ts-node scripts/sync-secrets.ts --env staging --create-env", - "secrets:prod": "ts-node scripts/sync-secrets.ts --env prod --create-env" + "secrets:prod": "ts-node scripts/sync-secrets.ts --env prod --create-env", + "pretest": "node --test dev/check-watch.cjs" }, "devDependencies": { "@types/body-parser": "^1.19.6", @@ -80,7 +84,9 @@ "@types/node": "^26.6.1", "@types/prettyjson": "^0.0.33", "@typescript/native": "npm:typescript@7.0.2", + "anymatch": "^3.1.3", "ejs": "^6.0.1", + "is-glob": "^4.0.3", "jest": "^30.5.1", "nodemon": "^3.1.14", "oxlint": "^1.83.0", diff --git a/infra/wab/README.md b/infra/wab/README.md index 38ca0ccb3..5bbae44fd 100644 --- a/infra/wab/README.md +++ b/infra/wab/README.md @@ -5,6 +5,12 @@ Welcome to the **Wallet Authentication Backend (WAB)** project! This README prov See [Service Resource Profiles](../../docs/reference/service-resource-profiles.md) for bounded defaults, database sizing, and HPA prerequisites. +The Dockerfile uses the governed, digest-pinned Node base from Docker Hub, matching +the other TS Stack services. This preserves the reviewed image bytes while +avoiding public ECR mirror throttling during builds. Base versions, equivalent +registry references and runtime package pins remain governed by +[`governance/container-images.json`](../../governance/container-images.json). + --- ## What Is the WAB? @@ -658,3 +664,23 @@ To contribute: ## License This project is available under the [Open BSV License Version 6](./LICENSE.txt). + +## Development watcher + +`npm run dev` uses the locked Nodemon CLI with a source-owned Chokidar 4 +compatibility adapter. It watches TypeScript, existing JavaScript/JSON extensions, +new source files and `.env`, retains the Node/ts-node telemetry preload, and +supports manual `rs` restarts. Existing ignored glob, directory, regex and +function options retain Chokidar 3 matching behavior. WAB replaces ts-node-dev +with this same CLI; production startup, HTTP contracts and persisted data are +unchanged. No public npm package version or consumer migration is required for +these standalone service development tools. + +The parent-scoped Chokidar substitution removes the affected braces dependency +without an advisory exclusion. The dated dependency registry owns its removal +condition. `npm test` first runs the actual locked watcher regression, including +clean shutdown, using the actual service development recipe. WAB also retains +compiler-configuration restarts. The basic UHRP service owns the adapter and +regression; the root service-copy generator synchronizes cloud UHRP and WAB. +Protected Linux image and exact-head CI checks must qualify release candidates; +source changes do not update deployed images. diff --git a/infra/wab/dev/bin/nodemon.js b/infra/wab/dev/bin/nodemon.js new file mode 100644 index 000000000..fe89254b7 --- /dev/null +++ b/infra/wab/dev/bin/nodemon.js @@ -0,0 +1,48 @@ +'use strict' + +const path = require('node:path') +const anymatch = require('anymatch') +const isGlob = require('is-glob') +const { bus } = require('nodemon/lib/utils') +const { rulesToMonitor } = require('nodemon/lib/monitor/match') + +function unix(value) { + const slashes = value.replaceAll('\\', '/') + return (slashes.startsWith('//') ? '/' : '') + slashes.replaceAll(/\/{2,}/g, '/') +} +function normalizeIgnored(value, cwd) { + if (typeof value !== 'string') return value + const joined = path.isAbsolute(value) ? value : path.join(cwd ?? '', value) + return unix(path.normalize(unix(joined))) +} + +/** Nodemon already expands watched globs into literal directories. Chokidar4 + * also needs the former glob-aware ignore predicate, including literal-directory + * recursion and custom ignored functions/regexes. Retain its published anymatch + * matcher instead of the removed braces-dependent watch-path expansion. */ +function applyOptions(config) { + const options = config.options.watchOptions ?? {} + let ignored + if (Object.hasOwn(options, 'ignored')) ignored = options.ignored + else { + ignored = rulesToMonitor([], Array.from(config.options.ignore), config).map(pattern => + pattern.slice(1) + ) + const dotFile = /[/\\]\./ + if (!config.dirs.some(directory => dotFile.test(directory))) ignored.push(dotFile) + } + const normalized = (Array.isArray(ignored) ? ignored : [ignored]).map(value => + normalizeIgnored(value, options.cwd) + ) + const directories = normalized + .filter(value => typeof value === 'string' && !isGlob(value)) + .map(value => value + '/**') + const predicate = anymatch([...normalized, ...directories], undefined, { dot: true }) + config.options.watchOptions = { ...options, ignored: (file, stats) => predicate([file, stats]) } +} + +module.exports = applyOptions +if (require.main === module) { + bus.on('config:update', applyOptions) + require('nodemon/bin/nodemon.js') +} diff --git a/infra/wab/dev/check-watch.cjs b/infra/wab/dev/check-watch.cjs new file mode 100644 index 000000000..7e380ce0e --- /dev/null +++ b/infra/wab/dev/check-watch.cjs @@ -0,0 +1,195 @@ +'use strict' + +const assert = require('node:assert/strict') +const fs = require('node:fs/promises') +const { readFileSync } = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { spawn } = require('node:child_process') +const { test } = require('node:test') +const applyOptions = require('./bin/nodemon.js') +const delay = ms => new Promise(resolve => setTimeout(resolve, ms)) + +test('development ignore options retain legacy glob, literal-directory, regex, function, cwd and dotfile behavior', () => { + const cases = [ + { ignored: '/repo/cache', file: '/repo/cache/child/file.ts', expected: true }, + { ignored: '/repo/cache', file: '/repo/cache-other/file.ts', expected: false }, + { ignored: 'cache', cwd: '/repo', file: '/repo/cache/file.ts', expected: true }, + { + ignored: String.raw`cache\nested`, + cwd: '/repo', + file: '/repo/cache/nested/file.ts', + expected: true + }, + { ignored: '**/generated/**', file: '/repo/generated/file.ts', expected: true }, + { ignored: '**/generated/**', file: '/repo/src/file.ts', expected: false }, + { ignored: ['**/*.ts', '!**/keep.ts'], file: '/repo/src/drop.ts', expected: true }, + { ignored: ['**/*.ts', '!**/keep.ts'], file: '/repo/src/keep.ts', expected: false }, + { ignored: /\.map$/, file: '/repo/file.map', expected: true }, + { ignored: /\.map$/, file: '/repo/file.ts', expected: false }, + { + ignored: (_file, stats) => stats?.marker === true, + file: '/repo/a.ts', + stats: { marker: true }, + expected: true + }, + { ignored: (_file, stats) => stats?.marker === true, file: '/repo/a.ts', expected: false }, + { ignored: undefined, file: '/repo/a.ts', expected: false }, + { ignored: [], file: '/repo/a.ts', expected: false } + ] + for (const item of cases) { + const config = { + dirs: ['/repo/src'], + options: { + ignore: [], + watchOptions: { ignored: item.ignored, cwd: item.cwd, usePolling: true, interval: 71 } + } + } + applyOptions(config) + assert.equal( + config.options.watchOptions.ignored(item.file, item.stats), + item.expected, + item.file + ) + assert.equal(config.options.watchOptions.usePolling, true) + assert.equal(config.options.watchOptions.interval, 71) + } + const defaults = { dirs: ['/repo/src'], options: { ignore: ['**/node_modules/**'] } } + applyOptions(defaults) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/.hidden.ts'), true) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/node_modules/pkg/a.ts'), true) + assert.equal(defaults.options.watchOptions.ignored('/repo/src/a.ts'), false) + const explicitDotfile = { dirs: ['/repo/.env', '/repo/src'], options: { ignore: [] } } + applyOptions(explicitDotfile) + assert.equal(explicitDotfile.options.watchOptions.ignored('/repo/.env'), false) +}) + +test( + 'actual locked watcher restarts TS, env and new source, retains preloads/manual restart, ignores dependencies and stops', + { timeout: 60000 }, + async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts-stack-dev-watch-')) + let child, + exited, + output = '', + errors = '', + forced = false + const starts = () => [...output.matchAll(/READY:(\d+):(\d+)/g)] + async function waitFor(predicate, description, remaining = 400) { + if (remaining === 0) throw new Error(`Watcher did not ${description}: ${output}\n${errors}`) + if (predicate()) return + if (child.exitCode !== null || child.signalCode !== null) + throw new Error(`Watcher exited during ${description}: ${errors}`) + await delay(25) + return waitFor(predicate, description, remaining - 1) + } + try { + await fs.mkdir(path.join(directory, 'src/node_modules'), { recursive: true }) + await fs.mkdir(path.join(directory, 'src/generated'), { recursive: true }) + await fs.symlink( + path.resolve(__dirname, '../node_modules'), + path.join(directory, 'node_modules'), + 'junction' + ) + await fs.writeFile( + path.join(directory, 'tsconfig.json'), + JSON.stringify({ + compilerOptions: { + module: 'commonjs', + target: 'es2022', + strict: true, + skipLibCheck: true, + types: ['node'] + } + }) + ) + await fs.writeFile(path.join(directory, '.env'), 'synthetic ordinary configuration') + await fs.writeFile(path.join(directory, 'src/dependency.ts'), 'export const value = 1') + await fs.writeFile( + path.join(directory, 'src/telemetry.ts'), + "console.log('PRELOAD:' + process.pid)" + ) + const recipe = JSON.parse(readFileSync(path.resolve(__dirname, '../package.json'), 'utf8')) + .scripts.dev + const prefix = 'node dev/bin/nodemon.js ' + assert.ok(recipe.startsWith(prefix) && recipe.endsWith('"')) + const [watch, execution] = recipe.slice(prefix.length, -1).split(' --exec "') + assert.ok(execution) + const options = watch.split(/\s+/) + const entry = execution.match(/src\/(index|server)\.ts$/)?.[0] + assert.ok(entry) + await fs.writeFile( + path.join(directory, entry), + "import { value } from './dependency'; console.log('READY:' + process.pid + ':' + value); setInterval(() => {}, 1000)" + ) + child = spawn( + process.execPath, + [ + path.join(__dirname, 'bin/nodemon.js'), + ...options, + '--ignore', + 'src/generated/**', + '--exec', + execution + ], + { + cwd: directory, + detached: true, + stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, CI: '1', NO_UPDATE_NOTIFIER: '1' } + } + ) + exited = new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + child.stdout.on('data', data => { + output += data.toString() + }) + child.stderr.on('data', data => { + errors += data.toString() + }) + await waitFor(() => starts().length === 1, 'start the synthetic TypeScript process') + await delay(500) + await fs.writeFile(path.join(directory, 'src/dependency.ts'), 'export const value = 2') + await waitFor(() => starts().length === 2, 'restart for a TypeScript dependency') + assert.equal(starts()[1][2], '2') + await delay(500) + await fs.writeFile(path.join(directory, 'src/node_modules/ignored.js'), 'module.exports = 1') + await fs.writeFile( + path.join(directory, 'src/generated/ignored.ts'), + 'export const ignored = 1' + ) + await delay(700) + assert.equal(starts().length, 2) + await fs.writeFile(path.join(directory, '.env'), 'synthetic changed configuration') + await waitFor(() => starts().length === 3, 'restart for the explicitly watched env file') + await delay(500) + await fs.writeFile(path.join(directory, 'src/new-file.ts'), 'export const newFile = 1') + await waitFor(() => starts().length === 4, 'restart for a new TypeScript source file') + child.stdin.write('rs\n') + await waitFor(() => starts().length === 5, 'accept a manual restart') + if (options.includes('tsconfig.json')) { + await delay(500) + await fs.appendFile(path.join(directory, 'tsconfig.json'), '\n') + await waitFor(() => starts().length === 6, 'restart for the watched compiler configuration') + } + assert.equal([...output.matchAll(/PRELOAD:/g)].length, starts().length) + } finally { + if (child?.exitCode === null && child.signalCode === null) { + process.kill(-child.pid, 'SIGTERM') + const escalation = setTimeout(() => { + forced = true + process.kill(-child.pid, 'SIGKILL') + }, 5000) + try { + await exited + } finally { + clearTimeout(escalation) + } + } + await fs.rm(directory, { recursive: true, force: true }) + } + assert.equal(forced, false, 'the CLI and owned child must exit without forced termination') + } +) diff --git a/infra/wab/package-lock.json b/infra/wab/package-lock.json index 06bae55de..e27f4b526 100644 --- a/infra/wab/package-lock.json +++ b/infra/wab/package-lock.json @@ -41,12 +41,14 @@ "@types/jest": "^30.0.0", "@types/node": "^26.6.1", "@typescript/native": "npm:typescript@7.0.2", + "anymatch": "^3.1.3", "body-parser": "^2.3.0", + "is-glob": "^4.0.3", "jest": "^30.5.1", + "nodemon": "^3.1.14", "oxlint": "^1.83.0", "ts-jest": "^29.4.12", "ts-node": "^10.9.2", - "ts-node-dev": "^2.0.0", "typescript": "npm:@typescript/typescript6@6.0.2" }, "engines": { @@ -3773,18 +3775,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/strip-bom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/@types/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-xevGOReSYGM7g/kUBZzPqCrR/KYAo+F0yiPc85WFTJa0MSLtyFTVTU6cJu/aV4mid7IffDIWqo69THF2o4JiEQ==", - "dev": true - }, - "node_modules/@types/strip-json-comments": { - "version": "0.0.30", - "resolved": "https://registry.npmjs.org/@types/strip-json-comments/-/strip-json-comments-0.0.30.tgz", - "integrity": "sha512-7NQmHra/JILCd1QqpSzl8+mJRc8ZHz3uDm8YV1Ks9IhK0epEiTw8aIErbvH9PI+6XbqhyIQy3462nEsn7UVzjQ==", - "dev": true - }, "node_modules/@types/yargs": { "version": "17.0.33", "resolved": "https://registry.npmjs.org/@types/yargs/-/yargs-17.0.33.tgz", @@ -4912,18 +4902,6 @@ "node": "*" } }, - "node_modules/binary-extensions": { - "version": "2.3.0", - "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", - "integrity": "sha512-Ceh+7ox5qe7LJuLHoY0feh3pHuUDHAcRUeyL2VYghZwfpkNIy/+8Ocg0a3UuSoYzavmylwuLWQOf3hl0jjMMIw==", - "dev": true, - "engines": { - "node": ">=8" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/bindings": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz", @@ -4992,18 +4970,6 @@ "node": "20 || >=22" } }, - "node_modules/braces": { - "version": "3.0.3", - "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz", - "integrity": "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==", - "dev": true, - "dependencies": { - "fill-range": "^7.1.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/browserslist": { "version": "4.28.9", "resolved": "https://registry.npmjs.org/browserslist/-/browserslist-4.28.9.tgz", @@ -5088,12 +5054,6 @@ "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", "license": "BSD-3-Clause" }, - "node_modules/buffer-from": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", - "integrity": "sha512-E+XQCRwSbaaiChtv6k6Dwgc+bx+Bs6vuKJHHl5kox/BaKbhiXzqQOwK4cO22yElGp2OCmjwVhT3HmxgyPGnJfQ==", - "dev": true - }, "node_modules/bytes": { "version": "3.1.2", "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", @@ -5216,30 +5176,6 @@ "node": ">=10" } }, - "node_modules/chokidar": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", - "integrity": "sha512-7VT13fmjotKpGipCW9JEQAusEPE+Ei8nl6/g4FBAmIm0GOOLMua9NDDo/DWp0ZAxCr3cPq5ZpBqmPAQgDda2Pw==", - "dev": true, - "dependencies": { - "anymatch": "~3.1.2", - "braces": "~3.0.2", - "glob-parent": "~5.1.2", - "is-binary-path": "~2.1.0", - "is-glob": "~4.0.1", - "normalize-path": "~3.0.0", - "readdirp": "~3.6.0" - }, - "engines": { - "node": ">= 8.10.0" - }, - "funding": { - "url": "https://paulmillr.com/funding/" - }, - "optionalDependencies": { - "fsevents": "~2.3.2" - } - }, "node_modules/chownr": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/chownr/-/chownr-3.0.0.tgz", @@ -5574,15 +5510,6 @@ "node": ">= 0.4" } }, - "node_modules/dynamic-dedupe": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/dynamic-dedupe/-/dynamic-dedupe-0.3.0.tgz", - "integrity": "sha512-ssuANeD+z97meYOqd50e04Ze5qp4bPqo8cCkI4TRjZkzAUgIDTrXV1R8QCdINpiI+hw14+rYazvTRdQrz0/rFQ==", - "dev": true, - "dependencies": { - "xtend": "^4.0.0" - } - }, "node_modules/eastasianwidth": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/eastasianwidth/-/eastasianwidth-0.2.0.tgz", @@ -5987,18 +5914,6 @@ "resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz", "integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==" }, - "node_modules/fill-range": { - "version": "7.1.1", - "resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz", - "integrity": "sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==", - "dev": true, - "dependencies": { - "to-regex-range": "^5.0.1" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/finalhandler": { "version": "2.1.1", "resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-2.1.1.tgz", @@ -6139,26 +6054,6 @@ "resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz", "integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==" }, - "node_modules/fs.realpath": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", - "integrity": "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==", - "dev": true - }, - "node_modules/fsevents": { - "version": "2.3.3", - "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.3.tgz", - "integrity": "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw==", - "dev": true, - "hasInstallScript": true, - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": "^8.16.0 || ^10.6.0 || >=11.0.0" - } - }, "node_modules/function-bind": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", @@ -6311,39 +6206,6 @@ "resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz", "integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==" }, - "node_modules/glob": { - "version": "7.2.3", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz", - "integrity": "sha512-nFR0zLpU2YCaRxwoCJvL6UvCH2JFyFVIvwTLsIf21AuHlMskA1hhTdk+LlYJtOlYt9v6dvszD2BGRqBL+iQK9Q==", - "deprecated": "Glob versions prior to v9 are no longer supported", - "dev": true, - "dependencies": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.1.1", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - }, - "engines": { - "node": "*" - }, - "funding": { - "url": "https://github.com/sponsors/isaacs" - } - }, - "node_modules/glob-parent": { - "version": "5.1.2", - "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", - "integrity": "sha512-AOIgSQCepiJYwP3ARnGx+5VnTu2HBYdzbGP45eLw1vr3zB3vZLeyed1sC9hnbcOc9/SrMyM5RPQrkGz4aS9Zow==", - "dev": true, - "dependencies": { - "is-glob": "^4.0.1" - }, - "engines": { - "node": ">= 6" - } - }, "node_modules/google-logging-utils": { "version": "1.1.3", "resolved": "https://registry.npmjs.org/google-logging-utils/-/google-logging-utils-1.1.3.tgz", @@ -6547,6 +6409,13 @@ } ] }, + "node_modules/ignore-by-default": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/ignore-by-default/-/ignore-by-default-1.0.1.tgz", + "integrity": "sha512-Ius2VYcGNk7T90CppJqcIkS5ooHUZyIQK+ClZfMfMNFEF9VSE73Fq+906u/CWu92x4gzZMWOwfFYckPObzdEbA==", + "dev": true, + "license": "ISC" + }, "node_modules/import-in-the-middle": { "version": "3.3.2", "resolved": "https://registry.npmjs.org/import-in-the-middle/-/import-in-the-middle-3.3.2.tgz", @@ -6590,17 +6459,6 @@ "node": ">=0.8.19" } }, - "node_modules/inflight": { - "version": "1.0.6", - "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", - "integrity": "sha512-k92I/b08q4wvFscXCLvqfsHCrjrF7yiXsQuIVvVE7N82W3+aqpzuUdBbfhWcy/FZR3/4IgflMgKLOsvPDrGCJA==", - "deprecated": "This module is not supported, and leaks memory. Do not use it. Check out lru-cache if you want a good and tested way to coalesce async requests by a key value, which is much more comprehensive and powerful.", - "dev": true, - "dependencies": { - "once": "^1.3.0", - "wrappy": "1" - } - }, "node_modules/inherits": { "version": "2.0.4", "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", @@ -6643,18 +6501,6 @@ "dev": true, "license": "MIT" }, - "node_modules/is-binary-path": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/is-binary-path/-/is-binary-path-2.1.0.tgz", - "integrity": "sha512-ZMERYes6pDydyuGidse7OsHxtbI7WVeUEozgR/g7rd0xUimYNlvZRE/K2MgZTjWy725IfelLeVcEM97mmtRGXw==", - "dev": true, - "dependencies": { - "binary-extensions": "^2.0.0" - }, - "engines": { - "node": ">=8" - } - }, "node_modules/is-core-module": { "version": "2.16.1", "resolved": "https://registry.npmjs.org/is-core-module/-/is-core-module-2.16.1.tgz", @@ -6708,15 +6554,6 @@ "node": ">=0.10.0" } }, - "node_modules/is-number": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/is-number/-/is-number-7.0.0.tgz", - "integrity": "sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==", - "dev": true, - "engines": { - "node": ">=0.12.0" - } - }, "node_modules/is-promise": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz", @@ -7951,19 +7788,6 @@ "url": "https://github.com/sponsors/sindresorhus" } }, - "node_modules/minimatch": { - "version": "3.1.5", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.5.tgz", - "integrity": "sha512-VgjWUsnnT6n+NUk6eZq77zeFdpW2LWDzP6zFGrCbHXiYNul5Dzqk2HHQ5uFH2DNW5Xbp8+jVzaeNt94ssEEl4w==", - "dev": true, - "license": "ISC", - "dependencies": { - "brace-expansion": "^1.1.7" - }, - "engines": { - "node": "*" - } - }, "node_modules/minimist": { "version": "1.2.8", "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz", @@ -7993,18 +7817,6 @@ "node": ">= 18" } }, - "node_modules/mkdirp": { - "version": "1.0.4", - "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz", - "integrity": "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==", - "dev": true, - "bin": { - "mkdirp": "bin/cmd.js" - }, - "engines": { - "node": ">=10" - } - }, "node_modules/mkdirp-classic": { "version": "0.5.3", "resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz", @@ -8239,6 +8051,117 @@ "node": ">=18" } }, + "node_modules/nodemon": { + "version": "3.1.14", + "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.14.tgz", + "integrity": "sha512-jakjZi93UtB3jHMWsXL68FXSAosbLfY0In5gtKq3niLSkrWznrVBzXFNOEMJUfc9+Ke7SHWoAZsiMkNP3vq6Jw==", + "dev": true, + "license": "MIT", + "dependencies": { + "chokidar": "^3.5.2", + "debug": "^4", + "ignore-by-default": "^1.0.1", + "minimatch": "^10.2.1", + "pstree.remy": "^1.1.8", + "semver": "^7.5.3", + "simple-update-notifier": "^2.0.0", + "supports-color": "^5.5.0", + "touch": "^3.1.0", + "undefsafe": "^2.0.5" + }, + "bin": { + "nodemon": "bin/nodemon.js" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/nodemon" + } + }, + "node_modules/nodemon/node_modules/chokidar": { + "version": "4.0.3", + "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-4.0.3.tgz", + "integrity": "sha512-Qgzu8kfBvo+cA4962jnP1KkS6Dop5NS6g7R5LFYJr4b8Ub94PPQXUksCw9PvXoeXPRRddRNC5C1JQUR2SMGtnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "readdirp": "^4.0.1" + }, + "engines": { + "node": ">= 14.16.0" + }, + "funding": { + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/nodemon/node_modules/has-flag": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", + "integrity": "sha512-sKJf1+ceQBr4SMkvQnBDNDtf4TXpVhVGateu0t918bl30FnbE2m4vNLX+VWe/dpjlb+HugGYzW7uQXH98HPEYw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/nodemon/node_modules/minimatch": { + "version": "10.2.6", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-10.2.6.tgz", + "integrity": "sha512-vpLQEs+VLCr1nU0BXS07maYoFwlDAH0gngQuuttxIwutDFEMHq2blX+8vpgxDdK3J1PwjCJiep77OitTZ4Ll1A==", + "dev": true, + "license": "BlueOak-1.0.0", + "dependencies": { + "brace-expansion": "^5.0.8" + }, + "engines": { + "node": "18 || 20 || >=22" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, + "node_modules/nodemon/node_modules/readdirp": { + "version": "4.1.2", + "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-4.1.2.tgz", + "integrity": "sha512-GDhwkLfywWL2s6vEjyhri+eXmfH6j1L7JE27WhqLeYzoh/A3DBaYGEj2H/HFZCn/kMfim73FXxEJTw06WtxQwg==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 14.18.0" + }, + "funding": { + "type": "individual", + "url": "https://paulmillr.com/funding/" + } + }, + "node_modules/nodemon/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/nodemon/node_modules/supports-color": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.5.0.tgz", + "integrity": "sha512-QjVjwdXIt408MIiAqCX4oUKsgU2EqAGzs2Ppkm4aQYbjm+ZEWEcW4SfFNTr4uMNZma0ey4f5lgLrkB0aX0QMow==", + "dev": true, + "license": "MIT", + "dependencies": { + "has-flag": "^3.0.0" + }, + "engines": { + "node": ">=4" + } + }, "node_modules/nopt": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/nopt/-/nopt-9.0.0.tgz", @@ -8486,15 +8409,6 @@ "node": ">=8" } }, - "node_modules/path-is-absolute": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", - "integrity": "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/path-key": { "version": "3.1.1", "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz", @@ -8816,6 +8730,13 @@ "node": ">=10" } }, + "node_modules/pstree.remy": { + "version": "1.1.8", + "resolved": "https://registry.npmjs.org/pstree.remy/-/pstree.remy-1.1.8.tgz", + "integrity": "sha512-77DZwxQmxKnu3aR542U+X8FypNzbfJ+C5XQDk3uWjWxn6151aIMGthWYRXTqT1E5oJvg+ljaa2OJi+VfvCOQ8w==", + "dev": true, + "license": "MIT" + }, "node_modules/pump": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/pump/-/pump-3.0.2.tgz", @@ -8922,31 +8843,6 @@ "node": ">= 6" } }, - "node_modules/readdirp": { - "version": "3.6.0", - "resolved": "https://registry.npmjs.org/readdirp/-/readdirp-3.6.0.tgz", - "integrity": "sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==", - "dev": true, - "dependencies": { - "picomatch": "^2.2.1" - }, - "engines": { - "node": ">=8.10.0" - } - }, - "node_modules/readdirp/node_modules/picomatch": { - "version": "2.3.2", - "resolved": "https://registry.npmjs.org/picomatch/-/picomatch-2.3.2.tgz", - "integrity": "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=8.6" - }, - "funding": { - "url": "https://github.com/sponsors/jonschlinkert" - } - }, "node_modules/real-require": { "version": "0.2.0", "resolved": "https://registry.npmjs.org/real-require/-/real-require-0.2.0.tgz", @@ -9339,6 +9235,32 @@ "simple-concat": "^1.0.0" } }, + "node_modules/simple-update-notifier": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/simple-update-notifier/-/simple-update-notifier-2.0.0.tgz", + "integrity": "sha512-a2B9Y0KlNXl9u/vsW6sTIu9vGEpfKu2wRV6l1H3XEas/0gUIzGzBoP/IouTcUQbm9JWZLH3COxyn03TYlFax6w==", + "dev": true, + "license": "MIT", + "dependencies": { + "semver": "^7.5.3" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/simple-update-notifier/node_modules/semver": { + "version": "7.8.5", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.5.tgz", + "integrity": "sha512-Y7/KDsb8LjooZpwaqGyulO6DQlksgCncchHGk+sZIY4SBvUocMBEFH5Ur1fI4dV+Jvl0w6cjvucaIi40puRioA==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, "node_modules/slash": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/slash/-/slash-3.0.0.tgz", @@ -9366,16 +9288,6 @@ "node": ">=0.10.0" } }, - "node_modules/source-map-support": { - "version": "0.5.13", - "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz", - "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==", - "dev": true, - "dependencies": { - "buffer-from": "^1.0.0", - "source-map": "^0.6.0" - } - }, "node_modules/split2": { "version": "4.2.0", "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", @@ -9832,18 +9744,6 @@ "url": "https://github.com/sponsors/SuperchupuDev" } }, - "node_modules/to-regex-range": { - "version": "5.0.1", - "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz", - "integrity": "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==", - "dev": true, - "dependencies": { - "is-number": "^7.0.0" - }, - "engines": { - "node": ">=8.0" - } - }, "node_modules/toidentifier": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz", @@ -9852,13 +9752,14 @@ "node": ">=0.6" } }, - "node_modules/tree-kill": { - "version": "1.2.2", - "resolved": "https://registry.npmjs.org/tree-kill/-/tree-kill-1.2.2.tgz", - "integrity": "sha512-L0Orpi8qGpRG//Nd+H90vFB+3iHnue1zSSGmNOOCh1GLJ7rUKVwV2HvijphGQS2UmhUZewS9VgvxYIdgr+fG1A==", + "node_modules/touch": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/touch/-/touch-3.1.1.tgz", + "integrity": "sha512-r0eojU4bI8MnHr8c5bNo7lJDdI2qXlWWJk6a9EAFG7vbhTjElYhBVS3/miuE0uOuoLdb8Mc/rVfsmm6eo5o9GA==", "dev": true, + "license": "ISC", "bin": { - "tree-kill": "cli.js" + "nodetouch": "bin/nodetouch.js" } }, "node_modules/ts-jest": { @@ -9984,84 +9885,6 @@ } } }, - "node_modules/ts-node-dev": { - "version": "2.0.0", - "resolved": "https://registry.npmjs.org/ts-node-dev/-/ts-node-dev-2.0.0.tgz", - "integrity": "sha512-ywMrhCfH6M75yftYvrvNarLEY+SUXtUvU8/0Z6llrHQVBx12GiFk5sStF8UdfE/yfzk9IAq7O5EEbTQsxlBI8w==", - "dev": true, - "license": "MIT", - "dependencies": { - "chokidar": "^3.5.1", - "dynamic-dedupe": "^0.3.0", - "minimist": "^1.2.6", - "mkdirp": "^1.0.4", - "resolve": "^1.0.0", - "rimraf": "^2.6.1", - "source-map-support": "^0.5.12", - "tree-kill": "^1.2.2", - "ts-node": "^10.4.0", - "tsconfig": "^7.0.0" - }, - "bin": { - "ts-node-dev": "lib/bin.js", - "tsnd": "lib/bin.js" - }, - "engines": { - "node": ">=0.8.0" - }, - "peerDependencies": { - "node-notifier": "*", - "typescript": "*" - }, - "peerDependenciesMeta": { - "node-notifier": { - "optional": true - } - } - }, - "node_modules/ts-node-dev/node_modules/rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "deprecated": "Rimraf versions prior to v4 are no longer supported", - "dev": true, - "dependencies": { - "glob": "^7.1.3" - }, - "bin": { - "rimraf": "bin.js" - } - }, - "node_modules/tsconfig": { - "version": "7.0.0", - "resolved": "https://registry.npmjs.org/tsconfig/-/tsconfig-7.0.0.tgz", - "integrity": "sha512-vZXmzPrL+EmC4T/4rVlT2jNVMWCi/O4DIiSj3UHg1OE5kCKbk4mfrXc6dZksLgRM/TZlKnousKH9bbTazUWRRw==", - "dev": true, - "dependencies": { - "@types/strip-bom": "^3.0.0", - "@types/strip-json-comments": "0.0.30", - "strip-bom": "^3.0.0", - "strip-json-comments": "^2.0.0" - } - }, - "node_modules/tsconfig/node_modules/strip-bom": { - "version": "3.0.0", - "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-3.0.0.tgz", - "integrity": "sha512-vavAMRXOgBVNF6nyEEmL3DBK19iRpDcoIwW+swQ+CbGiu7lju6t+JklA1MHweoWtadgt4ISVUsXLyDq34ddcwA==", - "dev": true, - "engines": { - "node": ">=4" - } - }, - "node_modules/tsconfig/node_modules/strip-json-comments": { - "version": "2.0.1", - "resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz", - "integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==", - "dev": true, - "engines": { - "node": ">=0.10.0" - } - }, "node_modules/tslib": { "version": "2.8.1", "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", @@ -10206,6 +10029,13 @@ "node": ">=0.8.0" } }, + "node_modules/undefsafe": { + "version": "2.0.5", + "resolved": "https://registry.npmjs.org/undefsafe/-/undefsafe-2.0.5.tgz", + "integrity": "sha512-WxONCrssBM8TSPRqN5EmsjVrsv4A8X12J4ArBiiayv3DyyG3ZlIg6yysuuSYdZsVz3TKcTg2fd//Ujd4CHV1iA==", + "dev": true, + "license": "MIT" + }, "node_modules/undici": { "version": "6.29.0", "resolved": "https://registry.npmjs.org/undici/-/undici-6.29.0.tgz", diff --git a/infra/wab/package.json b/infra/wab/package.json index 331a015a6..67d808eb3 100644 --- a/infra/wab/package.json +++ b/infra/wab/package.json @@ -6,7 +6,10 @@ "overrides": { "brace-expansion": "5.0.12", "gaxios": "7.3.0", - "js-yaml": "3.15.2" + "js-yaml": "3.15.2", + "nodemon": { + "chokidar": "4.0.3" + } }, "engines": { "node": ">=24 <25", @@ -16,12 +19,13 @@ "main": "dist/server.js", "scripts": { "start": "tsc && node --require ./dist/telemetry.js dist/server.js", - "dev": "ts-node-dev --respawn --require ./src/telemetry.ts src/server.ts", + "dev": "node dev/bin/nodemon.js --ext ts,js,mjs,cjs,json,env --watch .env --watch src --watch tsconfig.json --exec \"node -r ts-node/register -r ./src/telemetry.ts src/server.ts\"", "build": "tsc", - "lint": "oxlint src --deny-warnings", + "lint": "oxlint src dev --deny-warnings", "migrate": "knex --knexfile knexfile.ts migrate:latest", "test": "jest", - "test:coverage": "jest --coverage" + "test:coverage": "jest --coverage", + "pretest": "node --test dev/check-watch.cjs" }, "dependencies": { "@bsv/sdk": "^2.8.11", @@ -56,12 +60,14 @@ "@types/jest": "^30.0.0", "@types/node": "^26.6.1", "@typescript/native": "npm:typescript@7.0.2", + "anymatch": "^3.1.3", "body-parser": "^2.3.0", + "is-glob": "^4.0.3", "jest": "^30.5.1", + "nodemon": "^3.1.14", "oxlint": "^1.83.0", "ts-jest": "^29.4.12", "ts-node": "^10.9.2", - "ts-node-dev": "^2.0.0", "typescript": "npm:@typescript/typescript6@6.0.2" }, "license": "SEE LICENSE IN LICENSE.txt" diff --git a/packages/helpers/amountinator/README.md b/packages/helpers/amountinator/README.md index 792c09900..513a42668 100644 --- a/packages/helpers/amountinator/README.md +++ b/packages/helpers/amountinator/README.md @@ -79,3 +79,10 @@ Supported currencies: `BSV`, `SATS`, `USD`, `EUR`, `GBP`, `JPY`, `CNY`, `INR`, ` ## License This package is released under the [Open BSV License Version 6](./LICENSE.txt). + +## Next dependency release candidate + +This candidate refreshes the packed first-party dependency ranges for the next +wallet interoperability release. It adds no independent API or wire-format +change. Adopt after the new dependency graph is published; current wallet +releases retain their existing published pins. diff --git a/packages/helpers/bsv-wallet-helper/README.md b/packages/helpers/bsv-wallet-helper/README.md index c10af7bb7..5b4d2c1f4 100644 --- a/packages/helpers/bsv-wallet-helper/README.md +++ b/packages/helpers/bsv-wallet-helper/README.md @@ -21,28 +21,30 @@ npm install @bsv/wallet-helper ### Transaction Builder #### `TransactionBuilder` + Fluent transaction builder that simplifies creating BSV transactions with a clean, chainable API. ```typescript -import { TransactionBuilder } from '@bsv/wallet-helper'; +import { TransactionBuilder } from '@bsv/wallet-helper' // Simple P2PKH transaction with metadata -const result = await new TransactionBuilder(wallet, "Payment to Bob") - .addP2PKHOutput({ publicKey: bobPublicKey, satoshis: 1000, description: "Payment" }) - .addOpReturn(['APP_ID', JSON.stringify({ memo: 'Thanks!' })]) - .build(); +const result = await new TransactionBuilder(wallet, 'Payment to Bob') + .addP2PKHOutput({ publicKey: bobPublicKey, satoshis: 1000, description: 'Payment' }) + .addOpReturn(['APP_ID', JSON.stringify({ memo: 'Thanks!' })]) + .build() -console.log(`Transaction created: ${result.txid}`); +console.log(`Transaction created: ${result.txid}`) // Preview mode - see what will be sent without executing const preview = await new TransactionBuilder(wallet) .addP2PKHOutput({ publicKey: alicePublicKey, satoshis: 5000 }) - .build({ preview: true }); + .build({ preview: true }) -console.log('Transaction preview:', preview); +console.log('Transaction preview:', preview) ``` **Features:** + - Fluent API with method chaining - Support for P2PKH, Ordinal P2PKH, and custom outputs - **Automatic BRC-29 derivation** - omit addressOrParams to use secure random key derivation @@ -56,24 +58,26 @@ console.log('Transaction preview:', preview); 📖 **[Complete Documentation](./docs/transaction-builder.md)** **Example with automatic change:** + ```typescript // Change is automatically calculated: inputs - outputs - fees -await new TransactionBuilder(wallet, "Payment with change") - .addP2PKHInput({ sourceTransaction, sourceOutputIndex: 0, walletParams, description: "UTXO" }) - .addP2PKHOutput({ publicKey: recipientPublicKey, satoshis: 5000, description: "Payment" }) - .addChangeOutput({ walletParams, description: "Change" }) // Satoshis calculated automatically! - .build(); +await new TransactionBuilder(wallet, 'Payment with change') + .addP2PKHInput({ sourceTransaction, sourceOutputIndex: 0, walletParams, description: 'UTXO' }) + .addP2PKHOutput({ publicKey: recipientPublicKey, satoshis: 5000, description: 'Payment' }) + .addChangeOutput({ walletParams, description: 'Change' }) // Satoshis calculated automatically! + .build() ``` **Example with BRC-29 auto-derivation, basket, and customInstructions:** + ```typescript // Omit publicKey/walletParams to use automatic BRC-29 derivation // Derivation info is automatically added to customInstructions -await new TransactionBuilder(wallet, "Auto-derived transaction") - .addP2PKHOutput({ satoshis: 1000, description: "Payment" }) // Uses BRC-29 derivation - .basket("my-basket") // Set basket for this output - .customInstructions("app-specific-data") // Append custom instructions - .build(); +await new TransactionBuilder(wallet, 'Auto-derived transaction') + .addP2PKHOutput({ satoshis: 1000, description: 'Payment' }) // Uses BRC-29 derivation + .basket('my-basket') // Set basket for this output + .customInstructions('app-specific-data') // Append custom instructions + .build() // The output will have customInstructions with both app data and derivation info ``` @@ -81,52 +85,54 @@ await new TransactionBuilder(wallet, "Auto-derived transaction") ### Script Templates #### `WalletP2PKH` + Wallet-compatible Pay-to-Public-Key-Hash template. ```typescript -import { WalletP2PKH, type WalletDerivationParams } from '@bsv/wallet-helper'; +import { WalletP2PKH, type WalletDerivationParams } from '@bsv/wallet-helper' // Option 1: Direct public key -const p2pkh = new WalletP2PKH(); -const lockingScript = await p2pkh.lock({ publicKey: publicKeyHex }); +const p2pkh = new WalletP2PKH() +const lockingScript = await p2pkh.lock({ publicKey: publicKeyHex }) // Option 2: With BRC-100 wallet -const p2pkh = new WalletP2PKH(wallet); +const p2pkh = new WalletP2PKH(wallet) const lockingScript = await p2pkh.lock({ walletParams: { protocolID: [2, 'p2pkh'], keyID: '0', counterparty: 'self' } -}); +}) // Unlocking (requires wallet) const unlockingTemplate = p2pkh.unlock({ protocolID: [2, 'p2pkh'], keyID: '0', counterparty: 'self' -}); +}) ``` #### `WalletOrdP2PKH` + Wallet-compatible template for 1Sat Ordinals with inscription and MAP metadata support. ```typescript -import { WalletOrdP2PKH, type Inscription, type MAP } from '@bsv/wallet-helper'; +import { WalletOrdP2PKH, type Inscription, type MAP } from '@bsv/wallet-helper' // Create ordinal with inscription and metadata -const ordP2pkh = new WalletOrdP2PKH(wallet); +const ordP2pkh = new WalletOrdP2PKH(wallet) const inscription: Inscription = { dataB64: Buffer.from('Hello, Ordinals!').toString('base64'), contentType: 'text/plain' -}; +} const metadata: MAP = { app: 'my-app', type: 'greeting', author: 'Satoshi' -}; +} const lockingScript = await ordP2pkh.lock({ walletParams: { @@ -136,7 +142,7 @@ const lockingScript = await ordP2pkh.lock({ }, inscription, metadata -}); +}) ``` #### `WalletOrdLock` @@ -144,6 +150,7 @@ const lockingScript = await ordP2pkh.lock({ Wallet-compatible marketplace listing template for ordinals. An OrdLock output represents a listing with two spend paths: + - **Cancel** (seller): seller cancels their own listing using a wallet signature. - **Purchase** (buyer): buyer purchases the listing; the unlocking script commits to the final transaction outputs. @@ -167,7 +174,7 @@ await new TransactionBuilder(sellerWallet, 'Create listing') // Purchase listing (output ordering matters; see docs) await new TransactionBuilder(buyerWallet, 'Purchase listing') .addOrdLockInput({ sourceTransaction: listingTx, sourceOutputIndex: 0, kind: 'purchase' }) - .addP2PKHOutput({ publicKey: buyerPubKey, satoshis: 1 }) // Output 0: ordinal to buyer + .addP2PKHOutput({ publicKey: buyerPubKey, satoshis: 1 }) // Output 0: ordinal to buyer .addP2PKHOutput({ publicKey: sellerPubKey, satoshis: 1000 }) // Output 1: payment to seller .options({ randomizeOutputs: false }) .build() @@ -180,37 +187,40 @@ await new TransactionBuilder(buyerWallet, 'Purchase listing') ### Types #### `WalletDerivationParams` + Parameters for deriving keys from a BRC-100 wallet. ```typescript type WalletDerivationParams = { - protocolID: WalletProtocol; // e.g., [2, 'p2pkh'] - keyID: string; // e.g., '0' - counterparty: WalletCounterparty; // e.g., 'self' -}; + protocolID: WalletProtocol // e.g., [2, 'p2pkh'] + keyID: string // e.g., '0' + counterparty: WalletCounterparty // e.g., 'self' +} ``` **Note:** When wallet derivation parameters are omitted, the library uses the BRC-29 derivation scheme by default (using `brc29ProtocolID` from `@bsv/wallet-toolbox-client` and a randomly generated keyID), with `counterparty` defaulting to `'self'`. #### `Inscription` + 1Sat Ordinal inscription data. ```typescript type Inscription = { - dataB64: string; // Base64 encoded file data - contentType: string; // MIME type (e.g., 'image/png', 'text/plain') -}; + dataB64: string // Base64 encoded file data + contentType: string // MIME type (e.g., 'image/png', 'text/plain') +} ``` #### `MAP` + MAP (Magic Attribute Protocol) metadata for ordinals. ```typescript type MAP = { - app: string; // Application identifier (required) - type: string; // Data type identifier (required) - [key: string]: string; // Additional custom fields -}; + app: string // Application identifier (required) + type: string // Data type identifier (required) + [key: string]: string // Additional custom fields +} ``` ### Utilities @@ -220,6 +230,7 @@ Helper functions for wallet creation, transaction signing, script manipulation, 📖 **[Complete Utilities Documentation](./docs/utilities.md)** Includes: + - **Wallet Creation**: `makeWallet()` for creating BRC-100 wallets - **Transaction Signing**: `calculatePreimage()` for signature generation - **Script Utilities**: `addOpReturnData()` for adding metadata @@ -235,39 +246,39 @@ Includes: ### ✅ Correct Usage ```typescript -const wallet = await makeWallet('test', storageURL, privateKeyHex); -const p2pkh = new WalletP2PKH(wallet); +const wallet = await makeWallet('test', storageURL, privateKeyHex) +const p2pkh = new WalletP2PKH(wallet) const walletParams = { protocolID: [2, 'p2pkh'] as WalletProtocol, keyID: '0', counterparty: 'self' as WalletCounterparty -}; +} // Lock with wallet derivation -const lockingScript = await p2pkh.lock({ walletParams }); +const lockingScript = await p2pkh.lock({ walletParams }) // Unlock with SAME derivation params const unlockingTemplate = p2pkh.unlock({ protocolID: walletParams.protocolID, keyID: walletParams.keyID, counterparty: walletParams.counterparty -}); +}) ``` ### ❌ Incorrect Usage ```typescript // Lock with direct public key -const lockingScript = await p2pkh.lock({ publicKey: publicKeyHex }); +const lockingScript = await p2pkh.lock({ publicKey: publicKeyHex }) // Try to unlock with different derivation params // This WILL FAIL even if from same private key! const unlockingTemplate = p2pkh.unlock({ - protocolID: [2, 'different-protocol'], // Different protocol - keyID: '1', // Different keyID - counterparty: 'counterparty' // Different counterparty -}); + protocolID: [2, 'different-protocol'], // Different protocol + keyID: '1', // Different keyID + counterparty: 'counterparty' // Different counterparty +}) ``` **Why?** Each set of derivation parameters produces a different private key from the seed key -> different public key. The unlocking signature must match the exact public key hash used in the locking script. @@ -290,29 +301,29 @@ locally controlled. ```typescript // Recommended: Store params with your UTXO type MyUTXO = { - lockingScript: LockingScript; - satoshis: number; - derivationParams: WalletDerivationParams; // Store these! -}; + lockingScript: LockingScript + satoshis: number + derivationParams: WalletDerivationParams // Store these! +} // Later when spending const unlockingTemplate = p2pkh.unlock({ protocolID: utxo.derivationParams.protocolID, keyID: utxo.derivationParams.keyID, counterparty: utxo.derivationParams.counterparty -}); +}) ``` ## Examples For complete working examples, see the test files: -- **WalletP2PKH Examples**: [src/script-templates/__tests__/p2pkh.test.ts](./src/script-templates/__tests__/p2pkh.test.ts#L146) +- **WalletP2PKH Examples**: [src/script-templates/**tests**/p2pkh.test.ts](./src/script-templates/__tests__/p2pkh.test.ts#L146) - Creating and spending P2PKH transactions - Multiple inputs with wallet signing - Different signature scopes (SIGHASH_SINGLE) -- **WalletOrdP2PKH Examples**: [src/script-templates/__tests__/ordinal.test.ts](./src/script-templates/__tests__/ordinal.test.ts#L146) +- **WalletOrdP2PKH Examples**: [src/script-templates/**tests**/ordinal.test.ts](./src/script-templates/__tests__/ordinal.test.ts#L146) - Creating ordinals with inscriptions and metadata - Spending ordinal outputs - Reinscriptions (metadata-only updates) @@ -327,14 +338,14 @@ const original = await ordP2pkh.lock({ walletParams, inscription: { dataB64: largeImage, contentType: 'image/png' }, metadata: { app: 'gallery', type: 'art', owner: 'alice' } -}); +}) // Later: Update metadata only (saves transaction fees) const updated = await ordP2pkh.lock({ walletParams, // No inscription field = no file data metadata: { app: 'gallery', type: 'art', owner: 'bob', sold: 'true' } -}); +}) ``` ## License @@ -354,3 +365,10 @@ Contributions are welcome! Please open an issue or submit a pull request. This project is a work in progress and may change at any time. It is provided as-is, without any guarantees. Use at your own risk. + +## Next dependency release candidate + +This candidate refreshes the packed first-party dependency ranges for the next +wallet interoperability release. It adds no independent API or wire-format +change. Adopt after the new dependency graph is published; current wallet +releases retain their existing published pins. diff --git a/packages/helpers/fund-wallet/README.md b/packages/helpers/fund-wallet/README.md index 2f33ad032..56e8651f7 100644 --- a/packages/helpers/fund-wallet/README.md +++ b/packages/helpers/fund-wallet/README.md @@ -161,3 +161,10 @@ The recovered pre-monorepo ISC notice is retained in - [@bsv/sdk](https://www.npmjs.com/package/@bsv/sdk) - Bitcoin SV SDK - [@bsv/wallet-toolbox](https://www.npmjs.com/package/@bsv/wallet-toolbox) - Wallet management tools - [Metanet Desktop](https://metanet.bsvb.tech) - Local BSV wallet application + +## Next dependency release candidate + +This candidate refreshes the packed first-party dependency ranges for the next +wallet interoperability release. It adds no independent API or wire-format +change. Adopt after the new dependency graph is published; current wallet +releases retain their existing published pins. diff --git a/packages/helpers/simple/README.md b/packages/helpers/simple/README.md index d3abfd3a2..f49897306 100644 --- a/packages/helpers/simple/README.md +++ b/packages/helpers/simple/README.md @@ -142,3 +142,10 @@ const did = wallet.getDID() TS Stack first-party changes are under the [Open BSV License Version 6](./LICENSE.txt). The identified pre-monorepo source remains MIT-licensed; see [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md) and [LICENSES/](./LICENSES/). + +## Next dependency release candidate + +This candidate refreshes the packed first-party dependency ranges for the next +wallet interoperability release. It adds no independent API or wire-format +change. Adopt after the new dependency graph is published; current wallet +releases retain their existing published pins. diff --git a/packages/helpers/ts-templates/README.md b/packages/helpers/ts-templates/README.md index 4ccc307e9..760366464 100644 --- a/packages/helpers/ts-templates/README.md +++ b/packages/helpers/ts-templates/README.md @@ -25,6 +25,22 @@ tx.addOutput({ }) ``` +## Module compatibility + +Published 1.10.3 repairs CommonJS SDK import interop reported +in [stack#571](https://github.com/bsv-blockchain/ts-stack/issues/571). Both +`require('@bsv/templates')` and ESM imports construct and sign the same scripts; +root and `@bsv/templates/MandalaToken.ts` exports retain their public shape. +No API or script-encoding migration is required. + +The unpublished 1.10.4 candidate retains that runtime repair and adds the packed +consumer-check command. `pnpm pack:check` runs synthetic signed spends from exact packed artifacts in +clean CJS and ESM consumers with published SDK 2.8.0 and the candidate SDK. +For offline qualification, `TEMPLATES_PUBLISHED_SDK_TARBALL` may name a local +SDK 2.8.0 tarball. The check verifies the reviewed registry SHA-512 digest and +installed version for either the downloaded or supplied artifact; resolver age +policy and lifecycle-script restrictions stay unchanged. + ## Current Templates | Name | Description | diff --git a/packages/messaging/authsocket-client/README.md b/packages/messaging/authsocket-client/README.md index cb045de5e..fb40223ae 100644 --- a/packages/messaging/authsocket-client/README.md +++ b/packages/messaging/authsocket-client/README.md @@ -11,6 +11,14 @@ This package provides a **drop-in client-side solution** for Socket.IO that or any BRC-103-compatible server. - Minimal changes compared to normal `socket.io-client` usage. +## Next release candidate + +Unpublished 2.1.8 refreshes the UMD browser bundle with the integrated SDK. +AuthSocket APIs, signed event JSON and the ESM/CommonJS SDK peer range are +unchanged. The rebuilt UMD bytes differ from published 2.1.7, so distributors +should adopt the new artifact only after publication and retain its license +notices. Current wallet release pins are unaffected. + ## Installation Install the client and its required SDK peer: diff --git a/packages/messaging/message-box-client/CHANGELOG.md b/packages/messaging/message-box-client/CHANGELOG.md index b1f005ee8..eeda6f23d 100644 --- a/packages/messaging/message-box-client/CHANGELOG.md +++ b/packages/messaging/message-box-client/CHANGELOG.md @@ -13,6 +13,14 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +### 2.6.0 candidate — bounded PeerPay receive interoperability + +- Accept canonical standard base64 transaction strings alongside existing byte + arrays and numeric-key JSON objects in list, live and indexed PeerPay receipt. +- Bound encoded/decoded sizes before conversion and retain wallet validation and + acknowledgement order. Sender encoding and relay limits are unchanged. +- Keep #548 relay/preflight and #503 durable refund/outcome milestones open. + ### 2.6.0 candidate — list APIs keep payments the wallet did not store (#503) - `listMessages()` and `listMessagesLite()` now report `paymentOutcome` on each diff --git a/packages/messaging/message-box-client/README.md b/packages/messaging/message-box-client/README.md index 2de48947e..106754de6 100644 --- a/packages/messaging/message-box-client/README.md +++ b/packages/messaging/message-box-client/README.md @@ -316,6 +316,14 @@ Its SDK peer accepts both historical `number[]` and binary Wallet Wire `Uint8Array` transaction results, while payment messages retain a portable JSON byte-array representation. Receipt remains compatible with pending messages whose typed-array bytes were already serialized as contiguous numeric keys. +PeerPay receipt also accepts the canonical standard-base64 Atomic BEEF string +described by BRC-29. This receive-only compatibility applies to inbox lists, +indexed reloads, and live payments; senders keep the existing portable JSON +byte array. Empty, unpadded, URL-safe, whitespace-bearing, noncanonical pad-bit, +and malformed base64 strings are rejected before wallet work. Both encodings +retain the 64 MiB decoded transaction ceiling; base64 length is checked before +decoding. This does not raise relay request limits or provide delivery-limit +discovery, pre-broadcast sizing, or a durable retry/refund journal. The same compatibility contract covers paid-message fees, batch delivery, token settlements, live-message fallback, and generic remittance transport. In a multi-recipient send, the quoted server delivery fee applies to every diff --git a/packages/messaging/message-box-client/src/PeerPayClient.ts b/packages/messaging/message-box-client/src/PeerPayClient.ts index e6e672c20..e117bb0bf 100644 --- a/packages/messaging/message-box-client/src/PeerPayClient.ts +++ b/packages/messaging/message-box-client/src/PeerPayClient.ts @@ -37,6 +37,7 @@ import { type PeerMessage } from './types.js' import * as Logger from './Utils/logger.js' +import { decodePeerPayTransaction } from './Utils/peerPayTransaction.js' function hexToBytes(hex: string): number[] { if (!/^[0-9a-f]{64}$/.test(hex)) { @@ -143,13 +144,16 @@ function normalizePaymentParams(value: unknown): PaymentParams { return { recipient, amount: payment.amount } } -function normalizePaymentToken(value: unknown): PaymentToken { +function normalizePaymentToken(value: unknown, allowBase64 = false): PaymentToken { const token = dataRecord(value) const customInstructions = dataRecord(token?.customInstructions) if (token == null || customInstructions == null) { throw new TypeError('Incoming payment token is invalid') } - const transaction = normalizeBRC100ByteArray(token.transaction) + const transaction = + allowBase64 && typeof token.transaction === 'string' + ? decodePeerPayTransaction(token.transaction, MAX_PAYMENT_TRANSACTION_BYTES) + : normalizeBRC100ByteArray(token.transaction) if ( transaction == null || transaction.length === 0 || @@ -207,7 +211,7 @@ function paymentFromMessage(value: unknown): IncomingPayment | null { return { messageId: boundedMessageId(message.messageId), sender: canonicalIdentityKey(message.sender), - token: normalizePaymentToken(payment) + token: normalizePaymentToken(payment, true) } } catch { return null diff --git a/packages/messaging/message-box-client/src/Utils/peerPayTransaction.ts b/packages/messaging/message-box-client/src/Utils/peerPayTransaction.ts new file mode 100644 index 000000000..f94b8ac79 --- /dev/null +++ b/packages/messaging/message-box-client/src/Utils/peerPayTransaction.ts @@ -0,0 +1,30 @@ +import { base64ToArray } from '@bsv/sdk/primitives/utils' + +const alphabet = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/' + +/** BRC-29 receive boundary only: reject noncanonical or oversized text before decoding. */ +export function decodePeerPayTransaction(value: string, maximumBytes: number): number[] { + const invalid = (): never => { + throw new TypeError('Incoming payment transaction must be bounded canonical base64') + } + if ( + !Number.isSafeInteger(maximumBytes) || + maximumBytes < 1 || + value.length === 0 || + value.length % 4 !== 0 || + value.length > Math.ceil(maximumBytes / 3) * 4 + ) + invalid() + let padding = 0 + if (value.endsWith('==')) padding = 2 + else if (value.endsWith('=')) padding = 1 + const decodedLength = (value.length / 4) * 3 - padding + if (decodedLength === 0 || decodedLength > maximumBytes) invalid() + let last = 0 + for (let index = 0; index < value.length - padding; index++) { + last = alphabet.indexOf(value[index]) + if (last < 0) invalid() + } + if ((padding === 2 && (last & 15) !== 0) || (padding === 1 && (last & 3) !== 0)) invalid() + return base64ToArray(value) +} diff --git a/packages/messaging/message-box-client/src/__tests/PeerClients.validation.security.test.ts b/packages/messaging/message-box-client/src/__tests/PeerClients.validation.security.test.ts index 4eeb5ae86..fdcbe95d6 100644 --- a/packages/messaging/message-box-client/src/__tests/PeerClients.validation.security.test.ts +++ b/packages/messaging/message-box-client/src/__tests/PeerClients.validation.security.test.ts @@ -1,4 +1,4 @@ -import { PrivateKey, type WalletInterface } from '@bsv/sdk' +import { PrivateKey, PublicKey, P2PKH, Transaction, type WalletInterface } from '@bsv/sdk' import { jest } from '@jest/globals' import { PeerPayClient } from '../PeerPayClient.js' import { PeerTokenClient } from '../PeerTokenClient.js' @@ -158,6 +158,63 @@ describe('peer payment boundary validation', () => { ) }) + it.each(['AQID', [1, 2, 3], { 0: 1, 1: 2, 2: 3 }])( + 'normalizes BRC-29 and deployed byte representations across list and live paths %#', + async transaction => { + const incoming = message({ ...paymentToken, transaction }) + jest.spyOn(client, 'listMessages').mockResolvedValue([incoming]) + expect((await client.listIncomingPayments())[0].token.transaction).toEqual([1, 2, 3]) + const listen = jest.spyOn(client, 'listenForLiveMessages').mockResolvedValue() + const onPayment = jest.fn() + await client.listenForLivePayments({ onPayment }) + listen.mock.calls[0][0].onMessage(incoming) + expect(onPayment).toHaveBeenCalledWith( + expect.objectContaining({ + token: expect.objectContaining({ transaction: [1, 2, 3] }) + }) + ) + } + ) + + it('reloads a base64 Atomic BEEF and validates the actual payment before internalization', async () => { + const tx = new Transaction() + tx.addOutput({ + satoshis: 1, + lockingScript: new P2PKH().lock(PublicKey.fromString(identityA).toHash()) + }) + const bytes = tx.toAtomicBEEF() + jest + .spyOn(client, 'listMessagesLite') + .mockResolvedValue([ + message({ ...paymentToken, transaction: Buffer.from(bytes).toString('base64') }) + ]) + mockWallet.internalizeAction = jest.fn().mockResolvedValue({ accepted: true }) + const acknowledge = jest.spyOn(client, 'acknowledgeMessage').mockResolvedValue('ok') + await client.acceptPayment({ messageId: 'message-1', sender: identityA, token: paymentToken }) + expect(mockWallet.internalizeAction).toHaveBeenCalledWith( + expect.objectContaining({ tx: bytes }), + undefined + ) + expect(acknowledge).toHaveBeenCalledTimes(1) + }) + + it.each(['AR==', 'AQJ=', ' AQID', 'AQID\n', '-_8=', 'AQI', ''])( + 'retains invalid base64 %j without wallet or acknowledgement work', + async transaction => { + jest + .spyOn(client, 'listMessagesLite') + .mockResolvedValue([message({ ...paymentToken, transaction })]) + mockWallet.internalizeAction = jest.fn() + const acknowledge = jest.spyOn(client, 'acknowledgeMessage') + await expect( + client.acceptPayment({ messageId: 'message-1', sender: identityA, token: paymentToken }) + ).rejects.toThrow('not present exactly once') + expect(mockWallet.getPublicKey).not.toHaveBeenCalled() + expect(mockWallet.internalizeAction).not.toHaveBeenCalled() + expect(acknowledge).not.toHaveBeenCalled() + } + ) + it('bounds every payment collection before parsing attacker-controlled rows', async () => { const oversized = Array.from({ length: 1_001 }, () => message(paymentToken)) const list = jest.spyOn(client, 'listMessages') diff --git a/packages/messaging/message-box-client/src/__tests/peerPayTransaction.test.ts b/packages/messaging/message-box-client/src/__tests/peerPayTransaction.test.ts new file mode 100644 index 000000000..f14d57b61 --- /dev/null +++ b/packages/messaging/message-box-client/src/__tests/peerPayTransaction.test.ts @@ -0,0 +1,52 @@ +import fc from 'fast-check' +import { decodePeerPayTransaction } from '../Utils/peerPayTransaction.js' + +describe('BRC-29 canonical base64 receive boundary', () => { + it.each([ + ['AQ==', [1]], + ['AQI=', [1, 2]], + ['AQID', [1, 2, 3]], + ['+/8=', [251, 255]], + ['AAAA', [0, 0, 0]] + ])('decodes shared wire vector %s', (wire, expected) => { + expect(decodePeerPayTransaction(wire as string, 3)).toEqual(expected) + }) + + it.each([ + '', + 'A', + 'AQ', + 'AQ=', + 'AQ===', + '=AAA', + 'A=AA', + 'AA=A', + '====', + 'AR==', + 'AQJ=', + 'AQ==\n', + ' AQ==', + 'AQ I', + '-_8=', + 'ＡQ==' + ])('rejects %j', wire => { + expect(() => decodePeerPayTransaction(wire, 3)).toThrow('canonical base64') + }) + + it('checks decoded length as well as encoded length, before allocation', () => { + expect(() => decodePeerPayTransaction('AQID', 2)).toThrow('canonical base64') + expect(() => decodePeerPayTransaction('AQIDAQ==', 3)).toThrow('canonical base64') + expect(decodePeerPayTransaction('AQI=', 2)).toEqual([1, 2]) + }) + + it('matches an independent RFC 4648 encoder over all byte values and lengths', () => { + fc.assert( + fc.property(fc.uint8Array({ minLength: 1, maxLength: 4096 }), bytes => { + expect( + decodePeerPayTransaction(Buffer.from(bytes).toString('base64'), bytes.length) + ).toEqual(Array.from(bytes)) + }), + { numRuns: 200 } + ) + }) +}) diff --git a/packages/middleware/402-pay/CHANGELOG.md b/packages/middleware/402-pay/CHANGELOG.md index 1bf1d5f16..1780875c6 100644 --- a/packages/middleware/402-pay/CHANGELOG.md +++ b/packages/middleware/402-pay/CHANGELOG.md @@ -2,6 +2,11 @@ ## [Unreleased] +### 0.3.3 candidate — protocol selection documentation + +- Clarify that BRC-118 extends the separate SDK AuthFetch/BRC-105 middleware path. + This package's BRC-121 runtime, headers and replay behavior remain unchanged. + ### 0.3.2 candidate — replay-safe Atomic BEEF payments ### Maintenance diff --git a/packages/middleware/402-pay/README.md b/packages/middleware/402-pay/README.md index 21509f50c..e72dfcaa0 100644 --- a/packages/middleware/402-pay/README.md +++ b/packages/middleware/402-pay/README.md @@ -2,6 +2,14 @@ [BRC-121](https://github.com/bitcoin-sv/BRCs/blob/master/payments/0121.md) Simple 402 Payments -- server middleware and client for BSV micropayments over HTTP. +## Protocol boundary + +This package implements the BRC-121 `x-bsv-beef`/sender/nonce/time/vout contract. +BRC-118 extends the separate authenticated BRC-105 payment path in SDK AuthFetch +and `@bsv/payment-express-middleware`; it is not implicitly negotiated here. +See the [BRC-118 guide](../../../docs/guides/brc118-payments.md) when selecting an +integration. Existing 402-pay wire behavior is unchanged. + ## Install ```sh @@ -48,13 +56,7 @@ import { validatePayment, send402 } from '@bsv/402-pay/server' // In any HTTP handler: const requiredSatoshis = 100 -const result = await validatePayment( - req, - wallet, - requiredSatoshis, - 30_000, - sharedAtomicReplayStore -) +const result = await validatePayment(req, wallet, requiredSatoshis, 30_000, sharedAtomicReplayStore) if (!result) { send402(res, serverIdentityKey, requiredSatoshis) return diff --git a/packages/middleware/auth-express-middleware/README.md b/packages/middleware/auth-express-middleware/README.md index b191536e6..4fcbe95e4 100644 --- a/packages/middleware/auth-express-middleware/README.md +++ b/packages/middleware/auth-express-middleware/README.md @@ -35,6 +35,32 @@ need matching capacity; SDK 2.8.5 raises its header limits by 4x. No BRC100 call wire or wallet-data migration is required. Source publication is a separate protected release step. +## Exact-byte multipart authentication + +Set `captureRawBody: true` and install this middleware **before body parsers** +when composing BRC-118 payments. It collects bounded raw bytes, authenticates the +whole body and exact multipart Content-Type (including boundary), then exposes +`req.rawBody` and the verified `req.auth.supportsMultipart` marker. Existing +nonempty non-multipart signature preimages and the default parsed-body integration remain +compatible. Pair with payment middleware `enableMultipart: true`; raw auth alone +does not advertise payment support. Collection enforces request size, aggregate +memory, pending-request and timeout limits before authentication. See the +[BRC-118 guide](../../../docs/guides/brc118-payments.md) for limits, CORS and migration. + +## Empty binary request bodies + +Auth middleware 2.3.0 signs zero-length dense byte arrays and `Uint8Array`/`Buffer` +values with BRC-104's `-1` body-length sentinel, matching the integrated SDK AuthFetch 3.0.0 candidate. +This includes `express.raw({ type: 'application/octet-stream' })` mounted before +authentication without `captureRawBody`: Express may expose an empty Buffer even +when HTTP carries no body. Nonempty byte bodies keep their existing preimages. +Typed-array framing uses the same copied bytes for the length prefix and payload, +so a shadowed view-length property cannot produce an inconsistent frame. +For empty binary requests, upgrade the SDK client and auth receiver together; +older clients that sign length `0` or receivers that reconstruct length `0` +require the paired correction. No application payload or persisted-data migration +is required. + ## Requirements - Node.js 22 or newer diff --git a/packages/middleware/auth-express-middleware/package.json b/packages/middleware/auth-express-middleware/package.json index 269388db1..058cc28b2 100644 --- a/packages/middleware/auth-express-middleware/package.json +++ b/packages/middleware/auth-express-middleware/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/auth-express-middleware", - "version": "2.2.9", + "version": "2.3.0", "sideEffects": false, "engines": { "node": ">=22" @@ -86,7 +86,7 @@ "mime-types": "^3.0.2" }, "peerDependencies": { - "@bsv/sdk": "^2.8.5 || ^3.0.0", + "@bsv/sdk": "^2.9.0 || ^3.0.0", "@types/express": ">=4.17.0 <6", "express": ">=4.18.0 <6" }, diff --git a/packages/middleware/auth-express-middleware/src/__tests/Brc118.vectors.test.ts b/packages/middleware/auth-express-middleware/src/__tests/Brc118.vectors.test.ts new file mode 100644 index 000000000..12670a4d8 --- /dev/null +++ b/packages/middleware/auth-express-middleware/src/__tests/Brc118.vectors.test.ts @@ -0,0 +1,44 @@ +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' +import { Utils } from '@bsv/sdk' +import { + writeBodyToWriter, + writeRequestHeadersToWriter, + writeUrlToWriter +} from '../authMiddlewareHelpers.js' + +const corpus = JSON.parse( + readFileSync(resolve(process.cwd(), '../../../conformance/vectors/payments/brc118.json'), 'utf8') +) as { + vectors: Array<{ + id: string + input: { + request_id_hex: string + method: string + path: string + query: string + headers: Record + } + expected: { body_base64: string; request_preimage_hex: string } + }> +} + +describe('BRC-118 independently generated request preimages', () => { + it.each(corpus.vectors)('$id', vector => { + const writer = new Utils.Writer() + writer.write(Utils.toArray(vector.input.request_id_hex, 'hex')) + writer.writeVarIntNum(vector.input.method.length) + writer.write(Utils.toArray(vector.input.method, 'utf8')) + writeUrlToWriter( + new URL(`https://fixture.invalid${vector.input.path}${vector.input.query}`), + writer + ) + const req = { + headers: vector.input.headers, + body: Buffer.from(vector.expected.body_base64, 'base64') + } as never + writeRequestHeadersToWriter(req, writer) + writeBodyToWriter(req, writer) + expect(Utils.toHex(writer.toArray())).toBe(vector.expected.request_preimage_hex) + }) +}) diff --git a/packages/middleware/auth-express-middleware/src/__tests/RawAuthMiddleware.test.ts b/packages/middleware/auth-express-middleware/src/__tests/RawAuthMiddleware.test.ts new file mode 100644 index 000000000..778c2820d --- /dev/null +++ b/packages/middleware/auth-express-middleware/src/__tests/RawAuthMiddleware.test.ts @@ -0,0 +1,251 @@ +import { EventEmitter } from 'node:events' +import { PassThrough } from 'node:stream' +import { PrivateKey } from '@bsv/sdk' +import type { NextFunction, Response } from 'express' +import { createAuthMiddleware, ExpressTransport, type AuthRequest } from '../index.js' +import { RawRequestBodyError } from '../rawRequestBody.js' +import { MockWallet } from './MockWallet.js' + +const identityKey = new PrivateKey(1).toPublicKey().toString() + +function request(headers: Record = {}, path = '/paid') { + return Object.assign(new PassThrough(), { headers, path, method: 'POST', body: undefined }) +} + +function response() { + const res = Object.assign(new EventEmitter(), { + headersSent: false, + destroyed: false, + setHeader: jest.fn(), + status: jest.fn(), + json: jest.fn() + }) + res.status.mockReturnValue(res) + res.json.mockReturnValue(res) + return res +} + +async function flush() { + await new Promise(resolve => setImmediate(resolve)) +} + +describe('raw authentication middleware composition', () => { + afterEach(() => jest.restoreAllMocks()) + + it('rejects a truthy non-boolean raw-capture option', () => { + expect(() => + createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: 'true' as unknown as boolean + }) + ).toThrow('captureRawBody must be a boolean') + }) + + it('preserves exact bytes until verification completes, then decodes and advertises support', async () => { + let verified!: NextFunction + const transport = jest + .spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + .mockImplementation(async (req, _res, next) => { + expect(req.body).toEqual(Buffer.from(' { "message": "hello ☃" }\n')) + expect(req.rawBody).toBeUndefined() + expect(req.auth).toBeUndefined() + verified = next + }) + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true + }) + const req = request({ 'content-type': 'application/json' }) + const res = response() + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + req.end(Buffer.from(' { "message": "hello ☃" }\n')) + await flush() + expect(transport).toHaveBeenCalledTimes(1) + expect(next).not.toHaveBeenCalled() + const authRequest = req as unknown as AuthRequest + authRequest.auth = { identityKey } + verified() + expect(next).toHaveBeenCalledWith() + expect(authRequest.body).toEqual({ message: 'hello ☃' }) + expect(authRequest.rawBody).toEqual(Buffer.from(' { "message": "hello ☃" }\n')) + expect(authRequest.auth).toEqual({ identityKey, supportsMultipart: true }) + res.emit('finish') + }) + + it.each([undefined, 'unknown'])( + 'never advertises verified support for identity %s', + async identity => { + jest + .spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + .mockImplementation(async (req, _res, next) => { + if (identity !== undefined) req.auth = { identityKey: identity } + next() + }) + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true + }) + const req = request({ 'content-type': 'text/plain' }) + const res = response() + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + req.end(Buffer.from('public body')) + await flush() + expect(next).toHaveBeenCalledWith() + expect(req.body).toBe('public body') + expect((req as unknown as AuthRequest).auth?.supportsMultipart).toBeUndefined() + res.emit('finish') + } + ) + + it.each([ + [ + 'multipart/form-data; boundary="outer"', + Buffer.from('--outer\r\nopaque payload\r\n--outer--\r\n') + ], + ['application/octet-stream', Buffer.from([0, 255, 1])], + [undefined, Buffer.from([0, 255, 1])], + [undefined, undefined] + ])('retains an opaque or absent body for %s (case %#)', async (contentType, bytes) => { + jest + .spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + .mockImplementation(async (req, _res, next) => { + req.auth = { identityKey } + next() + }) + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true + }) + const req = request(contentType === undefined ? {} : { 'content-type': contentType as string }) + const res = response() + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + req.end(bytes) + await flush() + expect(next).toHaveBeenCalledWith() + expect(req.body).toEqual(bytes) + expect((req as unknown as AuthRequest).rawBody).toEqual(bytes) + res.emit('finish') + }) + + it('applies configured raw-byte limits before authentication or application dispatch', async () => { + const transport = jest.spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true, + transportLimits: { maxRequestBytes: 3, requestTimeoutMs: 100, maxPendingRequests: 1 } + }) + const req = request({ 'content-length': '4' }) + const res = response() + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + await flush() + expect(transport).not.toHaveBeenCalled() + expect(next).not.toHaveBeenCalled() + expect(res.status).toHaveBeenCalledWith(413) + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'ERR_AUTH_BODY' })) + expect((req as unknown as AuthRequest).auth).toBeUndefined() + req.destroy() + }) + + it('rejects malformed authenticated JSON before calling the application', async () => { + jest + .spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + .mockImplementation(async (req, _res, next) => { + req.auth = { identityKey } + next() + }) + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true + }) + const req = request({ 'content-type': 'application/json' }) + const res = response() + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + req.end(Buffer.from('{')) + await flush() + expect(next).not.toHaveBeenCalled() + expect(res.status).toHaveBeenCalledWith(400) + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'ERR_AUTH_MALFORMED' })) + res.emit('finish') + }) + + it('decodes the bounded handshake envelope without marking it as an application payload', async () => { + const transport = jest + .spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + .mockImplementation(async (req, _res, next) => { + expect(req.body).toEqual({ messageType: 'initialRequest' }) + next() + }) + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true + }) + const req = request({ 'content-type': 'application/json' }, '/.well-known/auth') + const res = response() + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + req.end(Buffer.from('{"messageType":"initialRequest"}')) + await flush() + expect(transport).toHaveBeenCalledTimes(1) + expect(next).toHaveBeenCalledWith() + expect((req as unknown as AuthRequest).rawBody).toBeUndefined() + expect((req as unknown as AuthRequest).auth).toBeUndefined() + res.emit('finish') + }) + + it('forwards failed verification without decoding bytes or granting capability', async () => { + const failure = new Error('signature rejected') + jest + .spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + .mockImplementation(async (_req, _res, next) => next(failure)) + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true + }) + const req = request({ 'content-type': 'application/json' }) + const res = response() + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + req.end(Buffer.from('{"private":true}')) + await flush() + expect(next).toHaveBeenCalledWith(failure) + expect(req.body).toEqual(Buffer.from('{"private":true}')) + expect((req as unknown as AuthRequest).rawBody).toBeUndefined() + expect((req as unknown as AuthRequest).auth).toBeUndefined() + res.emit('finish') + }) + + it.each(['open', 'sent', 'destroyed'] as const)( + 'handles raw-body refusal with a %s response', + async state => { + const transport = jest.spyOn(ExpressTransport.prototype, 'handleIncomingRequest') + const middleware = createAuthMiddleware({ + wallet: new MockWallet(new PrivateKey(1)), + captureRawBody: true + }) + const req = request({ 'content-encoding': 'gzip' }) + const res = response() + res.headersSent = state === 'sent' + res.destroyed = state === 'destroyed' + const next = jest.fn() + middleware(req as unknown as AuthRequest, res as unknown as Response, next) + await flush() + expect(transport).not.toHaveBeenCalled() + if (state === 'open') { + expect(next).not.toHaveBeenCalled() + expect(res.setHeader).toHaveBeenCalledWith('Connection', 'close') + expect(res.status).toHaveBeenCalledWith(415) + expect(res.json).toHaveBeenCalledWith(expect.objectContaining({ code: 'ERR_AUTH_BODY' })) + } else { + expect(next).toHaveBeenCalledWith(expect.any(RawRequestBodyError)) + expect(res.json).not.toHaveBeenCalled() + expect(res.setHeader).not.toHaveBeenCalled() + } + req.destroy() + } + ) +}) diff --git a/packages/middleware/auth-express-middleware/src/__tests/RawRequestBody.test.ts b/packages/middleware/auth-express-middleware/src/__tests/RawRequestBody.test.ts new file mode 100644 index 000000000..e4f54d49a --- /dev/null +++ b/packages/middleware/auth-express-middleware/src/__tests/RawRequestBody.test.ts @@ -0,0 +1,108 @@ +import { PassThrough } from 'node:stream' +import { EventEmitter } from 'node:events' +import type { Request, Response } from 'express' +import { RawRequestBodyReader } from '../rawRequestBody.js' + +function request(headers: Record = {}) { + return Object.assign(new PassThrough(), { headers, body: undefined }) as unknown as Request +} +function response() { + return new EventEmitter() as unknown as Response +} + +describe('bounded pre-auth raw request collection', () => { + it('retains exact raw bytes without extracting or parsing payment data', async () => { + const reader = new RawRequestBodyReader(100, 1000, 2) + const req = request() + const res = response() + const done = reader.capture(req, res) + ;(req as unknown as PassThrough).end(Buffer.from('{ "value": 1 }\n')) + await done + expect(req.body).toEqual(Buffer.from('{ "value": 1 }\n')) + expect(reader.hasCaptured(req)).toBe(true) + res.emit('finish') + }) + + it.each([ + [{ 'content-length': '101' }, 413], + [{ 'content-length': '-1' }, 400], + [{ 'content-length': '9007199254740992' }, 400], + [{ 'content-encoding': 'gzip' }, 415] + ])('refuses an invalid or excessive declared body %#', async (headers, status) => { + const reader = new RawRequestBodyReader(100, 1000, 2) + await expect( + reader.capture(request(headers as Record), response()) + ).rejects.toMatchObject({ status }) + }) + + it('bounds streamed bytes independently of Content-Length and releases failed capacity', async () => { + const reader = new RawRequestBodyReader(3, 1000, 1) + const first = request() + const failure = expect(reader.capture(first, response())).rejects.toMatchObject({ status: 413 }) + ;(first as unknown as PassThrough).end(Buffer.from('four')) + await failure + const second = request() + const res = response() + const done = reader.capture(second, res) + ;(second as unknown as PassThrough).end(Buffer.from('ok')) + await done + res.emit('finish') + }) + + it('keeps completed-but-unanswered bodies inside both aggregate and request-count budgets', async () => { + const reader = new RawRequestBodyReader(8, 1000, 2, 8) + const first = request() + const firstRes = response() + const done = reader.capture(first, firstRes) + ;(first as unknown as PassThrough).end(Buffer.from('12345678')) + await done + const second = request() + const failure = expect(reader.capture(second, response())).rejects.toMatchObject({ + status: 503 + }) + ;(second as unknown as PassThrough).end(Buffer.from('a')) + await failure + firstRes.emit('finish') + const third = request() + const thirdRes = response() + const thirdDone = reader.capture(third, thirdRes) + ;(third as unknown as PassThrough).end(Buffer.from('12345678')) + await thirdDone + thirdRes.emit('close') + }) + + it('bounds slow uploads, disconnects, and pending requests without leaking listeners', async () => { + jest.useFakeTimers() + try { + const reader = new RawRequestBodyReader(100, 25, 1) + const slow = request() + const failure = expect(reader.capture(slow, response())).rejects.toMatchObject({ + status: 408 + }) + await expect(reader.capture(request(), response())).rejects.toMatchObject({ status: 503 }) + await jest.advanceTimersByTimeAsync(25) + await failure + expect(slow.listenerCount('data')).toBe(0) + const disconnected = request() + const res = response() + const aborted = expect(reader.capture(disconnected, res)).rejects.toMatchObject({ + status: 400 + }) + res.emit('close') + await aborted + expect(disconnected.listenerCount('data')).toBe(0) + expect(jest.getTimerCount()).toBe(0) + } finally { + jest.useRealTimers() + } + }) + + it('requires installation before a parser consumes the request', async () => { + const reader = new RawRequestBodyReader(100, 1000, 1) + const req = request() + req.body = { already: 'parsed' } + await expect(reader.capture(req, response())).rejects.toMatchObject({ status: 400 }) + expect(reader.hasCaptured(req)).toBe(false) + expect(() => new RawRequestBodyReader(-1, 1000, 1)).toThrow('finite') + }) +}) diff --git a/packages/middleware/auth-express-middleware/src/__tests/authMiddlewareHelpers.test.ts b/packages/middleware/auth-express-middleware/src/__tests/authMiddlewareHelpers.test.ts index deb5746df..b86954f0d 100644 --- a/packages/middleware/auth-express-middleware/src/__tests/authMiddlewareHelpers.test.ts +++ b/packages/middleware/auth-express-middleware/src/__tests/authMiddlewareHelpers.test.ts @@ -173,6 +173,44 @@ describe('auth middleware helpers', () => { expect(readBody(writer)).toEqual(expected) }) + it.each([ + ['dense byte array', []], + ['Uint8Array', new Uint8Array(0)], + ['Buffer', Buffer.alloc(0)], + ['empty subarray', new Uint8Array([0, 255]).subarray(1, 1)] + ])('writes the absent-body sentinel for an empty %s', (_name, body) => { + const writer = new Utils.Writer() + writeBodyToWriter( + { body, headers: { 'content-type': 'application/octet-stream' } } as any, + writer + ) + const reader = new Utils.Reader(writer.toArray()) + expect(reader.readVarIntNum()).toBe(-1) + expect(reader.pos).toBe(reader.bin.length) + }) + + it.each([ + ['Uint8Array', new Uint8Array([0, 128, 255]), [0, 128, 255]], + ['empty Uint8Array', new Uint8Array(0), []], + ['Buffer', Buffer.from([0, 128, 255]), [0, 128, 255]], + ['empty Buffer', Buffer.alloc(0), []] + ])( + 'frames copied %s bytes independently of shadowed length metadata', + (_name, body, expected) => { + const bytes = expected as number[] + Object.defineProperty(body, 'length', { value: bytes.length === 0 ? 3 : 0 }) + const writer = new Utils.Writer() + writeBodyToWriter( + { body, headers: { 'content-type': 'application/octet-stream' } } as any, + writer + ) + const reader = new Utils.Reader(writer.toArray()) + expect(reader.readVarIntNum()).toBe(bytes.length === 0 ? -1 : bytes.length) + expect(reader.read(bytes.length)).toEqual(bytes) + expect(reader.pos).toBe(reader.bin.length) + } + ) + it.each([ [undefined, undefined], ['', 'text/plain'], diff --git a/packages/middleware/auth-express-middleware/src/__tests/integration.test.ts b/packages/middleware/auth-express-middleware/src/__tests/integration.test.ts index 38f4aadc9..e7b24a7a4 100644 --- a/packages/middleware/auth-express-middleware/src/__tests/integration.test.ts +++ b/packages/middleware/auth-express-middleware/src/__tests/integration.test.ts @@ -156,6 +156,51 @@ describe('AuthFetch and AuthExpress Integration Tests', () => { await expect(authFetch.fetch(`${origin}/empty-404`)).rejects.toThrow(/signature/i) }) + describe.each(['POST', 'PUT'])('%s through express.raw without captureRawBody', method => { + test.each([ + ['dense byte array', [], []], + ['Uint8Array', new Uint8Array(0), []], + ['Buffer', Buffer.alloc(0), []], + ['absent body', undefined, []], + ['nonempty subarray', new Uint8Array([99, 0, 255, 88]).subarray(1, 3), [0, 255]] + ])('authenticates a %s body and the signed response', async (_name, body, expected) => { + const authFetch = new AuthFetch(new MockWallet(privKey)) + const route = method === 'PUT' ? '/put-endpoint' : '/other-endpoint' + const result = await authFetch.fetch(`${origin}${route}`, { + method, + headers: { 'content-type': 'application/octet-stream' }, + body: body as unknown as BodyInit + }) + expect(result.status).toBe(200) + expect(result.headers.get('x-bsv-auth-identity-key')).toBeTruthy() + const response = await result.json() + if (method === 'PUT') expect(response.body).toEqual(expected) + }) + }) + + test('rejects an empty octet-stream request changed to a nonempty body in transit', async () => { + const tamper: typeof fetch = async (url, init) => + fetch( + url, + String(url).endsWith('/put-endpoint') ? { ...init, body: new Uint8Array([1]) } : init + ) + const authFetch = new AuthFetch( + new MockWallet(privKey), + undefined, + undefined, + undefined, + {}, + tamper + ) + await expect( + authFetch.fetch(`${origin}/put-endpoint`, { + method: 'PUT', + headers: { 'content-type': 'application/octet-stream' }, + body: new Uint8Array(0) + }) + ).rejects.toThrow(/signature|authentication/i) + }) + test('Test 1: Simple POST request with JSON', async () => { const walletWithRequests = new MockWallet(privKey) const authFetch = new AuthFetch(walletWithRequests) diff --git a/packages/middleware/auth-express-middleware/src/__tests/testExpressServer.ts b/packages/middleware/auth-express-middleware/src/__tests/testExpressServer.ts index c0ee6fa4e..106a89a04 100644 --- a/packages/middleware/auth-express-middleware/src/__tests/testExpressServer.ts +++ b/packages/middleware/auth-express-middleware/src/__tests/testExpressServer.ts @@ -47,7 +47,7 @@ export const startServer = (_port = 3000): Server => { app.use(bodyParser.json()) app.use(express.urlencoded({ extended: true })) app.use(express.text()) - app.use(bodyParser.raw({ type: 'application/octet-stream', limit: '500mb' })) + app.use(express.raw({ type: 'application/octet-stream', limit: '500mb' })) // Mocked certificate and wallet setup // Used in the authentication middleware as needed: diff --git a/packages/middleware/auth-express-middleware/src/authMiddlewareHelpers.ts b/packages/middleware/auth-express-middleware/src/authMiddlewareHelpers.ts index 223b41e28..d5739bbe1 100644 --- a/packages/middleware/auth-express-middleware/src/authMiddlewareHelpers.ts +++ b/packages/middleware/auth-express-middleware/src/authMiddlewareHelpers.ts @@ -1,3 +1,4 @@ +import { authenticatedContentType } from '@bsv/sdk/auth/utils/paymentTransport' import { Writer, toArray } from '@bsv/sdk/primitives/utils' import { Request } from 'express' import { stringifyBRC100 } from '@bsv/sdk' @@ -127,7 +128,7 @@ export function writeRequestHeadersToWriter(req: Request, writer: Writer): void if (typeof v !== 'string') { throw new TypeError('Signed request headers must have one exact string value.') } - const headerValue = k === 'content-type' ? v.split(';')[0].trim() : v + const headerValue = k === 'content-type' ? authenticatedContentType(v) : v includedHeaders.push([k, headerValue]) } } @@ -196,16 +197,19 @@ export function writeBodyToWriter( const byteArray = copyDenseByteArray(body) if (byteArray !== undefined) { - writer.writeVarIntNum(byteArray.length) + // HTTP cannot distinguish an absent body from zero transmitted bytes. + // Match AuthFetch's BRC-104 empty-body sentinel after Express raw parsing. + writer.writeVarIntNum(byteArray.length === 0 ? -1 : byteArray.length) writer.write(byteArray) debugLog('[writeBodyToWriter] Body recognized as number[]', { length: byteArray.length }) return } if (body instanceof Uint8Array) { - writer.writeVarIntNum(body.length) - writer.write(Array.from(body)) - debugLog('[writeBodyToWriter] Body recognized as Uint8Array', { length: body.length }) + const bodyAsArray = Array.from(body) + writer.writeVarIntNum(bodyAsArray.length === 0 ? -1 : bodyAsArray.length) + writer.write(bodyAsArray) + debugLog('[writeBodyToWriter] Body recognized as Uint8Array', { length: bodyAsArray.length }) return } diff --git a/packages/middleware/auth-express-middleware/src/index.ts b/packages/middleware/auth-express-middleware/src/index.ts index c2011a787..2ee8f894e 100644 --- a/packages/middleware/auth-express-middleware/src/index.ts +++ b/packages/middleware/auth-express-middleware/src/index.ts @@ -1,3 +1,6 @@ +import { RawRequestBodyReader, RawRequestBodyError } from './rawRequestBody.js' +import { isMultipartPaymentType } from '@bsv/sdk/auth/utils/paymentTransport' +import { decodePaymentPayload } from '@bsv/sdk/auth/utils/decodePaymentPayload' import { Reader, Writer, toArray, toBase64, toHex, toUTF8 } from '@bsv/sdk/primitives/utils' import fs from 'node:fs' import path from 'node:path' @@ -90,7 +93,11 @@ export interface AuthTransportLimits { export interface AuthRequest extends Request { auth?: { identityKey: PubKeyHex + /** Raw-byte receiver support; set only after successful mutual authentication. */ + supportsMultipart?: boolean } + /** Exact authenticated application body. Payment middleware replaces this with the inner body. */ + rawBody?: Uint8Array } export interface CertificateApprovalStore { @@ -131,6 +138,8 @@ export class InMemoryCertificateApprovalStore implements CertificateApprovalStor // Developers may optionally provide a handler for incoming certificates. export interface AuthMiddlewareOptions { wallet: WalletInterface + /** Collect bounded raw bytes before auth. Install before any body parser. Required for BRC-118. */ + captureRawBody?: boolean // Optional session store. Default is in-process synchronous `SessionManager`. // Pass an `AsyncSessionManager` (Redis/SQL-backed, etc.) to share state // across load-balanced instances; Peer awaits internally so both work. @@ -1990,6 +1999,22 @@ function buildResponsePayload( return writer.toArray() } +function validateAuthMiddlewareSettings(options: AuthMiddlewareOptions): void { + const { allowUnauthenticated, logLevel, onCertificatesReceived } = options + if (allowUnauthenticated !== undefined && typeof allowUnauthenticated !== 'boolean') { + throw new TypeError('allowUnauthenticated must be a boolean.') + } + if (options.captureRawBody !== undefined && typeof options.captureRawBody !== 'boolean') { + throw new TypeError('captureRawBody must be a boolean.') + } + if (logLevel !== undefined && !(['debug', 'info', 'warn', 'error'] as const).includes(logLevel)) { + throw new TypeError('logLevel must be debug, info, warn, or error.') + } + if (onCertificatesReceived !== undefined && typeof onCertificatesReceived !== 'function') { + throw new TypeError('onCertificatesReceived must be a function.') + } +} + /** * Creates an Express middleware that handles authentication via BSV-SDK. * @@ -2022,15 +2047,7 @@ export function createAuthMiddleware(options: AuthMiddlewareOptions): RequestHan } throw new TypeError('You must configure the auth middleware with a wallet.') } - if (allowUnauthenticated !== undefined && typeof allowUnauthenticated !== 'boolean') { - throw new TypeError('allowUnauthenticated must be a boolean.') - } - if (logLevel !== undefined && !(['debug', 'info', 'warn', 'error'] as const).includes(logLevel)) { - throw new TypeError('logLevel must be debug, info, warn, or error.') - } - if (onCertificatesReceived !== undefined && typeof onCertificatesReceived !== 'function') { - throw new TypeError('onCertificatesReceived must be a function.') - } + validateAuthMiddlewareSettings(options) const transport = new ExpressTransport( allowUnauthenticated ?? false, @@ -2069,6 +2086,60 @@ export function createAuthMiddleware(options: AuthMiddlewareOptions): RequestHan transport.setPeer(peer) const telemetry = new Telemetry(telemetryConfig) + const rawReader = + options.captureRawBody === true + ? new RawRequestBodyReader( + transportLimits?.maxRequestBytes ?? DEFAULT_MAX_REQUEST_BYTES, + transportLimits?.requestTimeoutMs ?? 30_000, + transportLimits?.maxPendingRequests ?? 1_000 + ) + : undefined + const dispatch = async (req: AuthRequest, res: Response, next: NextFunction): Promise => { + try { + if (rawReader !== undefined) { + await rawReader.capture(req, res) + if (req.path === WELL_KNOWN_AUTH_PATH) { + req.body = decodePaymentPayload(req.body, 'application/json') + } + } + const verifiedNext: NextFunction = (error?: unknown): void => { + if (error !== undefined) { + next(error) + return + } + if (rawReader?.hasCaptured(req) === true && req.path !== WELL_KNOWN_AUTH_PATH) { + if (req.auth !== undefined && req.auth.identityKey !== 'unknown') + req.auth.supportsMultipart = true + req.rawBody = req.body + const contentType = req.headers['content-type'] + if (typeof contentType !== 'string' || !isMultipartPaymentType(contentType)) { + try { + req.body = decodePaymentPayload(req.rawBody, contentType) + } catch { + res.status(400).json({ + status: 'error', + code: 'ERR_AUTH_MALFORMED', + description: 'Invalid authenticated application body.' + }) + return + } + } + } + next() + } + await transport.handleIncomingRequest(req, res, verifiedNext, onCertificatesReceived) + } catch (error) { + if (error instanceof RawRequestBodyError && !res.headersSent && !res.destroyed) { + res.setHeader('Connection', 'close') + res.status(error.status).json({ + status: 'error', + code: 'ERR_AUTH_BODY', + description: 'The request body could not be accepted.' + }) + } else throw error + } + } + return (req, res, next) => { if (logger && logLevel && isLogLevelEnabled(logLevel, 'debug')) { getLogMethod(logger, 'debug')('[createAuthMiddleware] Incoming request to auth middleware', { @@ -2078,7 +2149,7 @@ export function createAuthMiddleware(options: AuthMiddlewareOptions): RequestHan }) } if (!telemetry.enabled) { - void transport.handleIncomingRequest(req, res, next, onCertificatesReceived).catch(next) + void dispatch(req, res, next).catch(next) return } @@ -2125,11 +2196,9 @@ export function createAuthMiddleware(options: AuthMiddlewareOptions): RequestHan end(res.writableEnded ? 'ok' : 'cancelled', undefined, 'connection_closed') }) - void transport - .handleIncomingRequest(req, res, tracedNext, onCertificatesReceived) - .catch(error => { - end('error', error, 'middleware_error') - next(error) - }) + void dispatch(req, res, tracedNext).catch(error => { + end('error', error, 'middleware_error') + next(error) + }) } } diff --git a/packages/middleware/auth-express-middleware/src/rawRequestBody.ts b/packages/middleware/auth-express-middleware/src/rawRequestBody.ts new file mode 100644 index 000000000..e54d964e6 --- /dev/null +++ b/packages/middleware/auth-express-middleware/src/rawRequestBody.ts @@ -0,0 +1,114 @@ +import type { Request, Response } from 'express' + +export class RawRequestBodyError extends Error { + constructor(readonly status: number) { + super('The raw authentication request could not be read.') + } +} + +/** Optional receiver-first raw collector. No multipart extraction occurs here. */ +export class RawRequestBodyReader { + private bufferedBytes = 0 + private pending = 0 + private readonly captured = new WeakSet() + + constructor( + private readonly maxRequestBytes: number, + private readonly timeoutMs: number, + private readonly maxPending: number, + private readonly maxBufferedBytes = 64 * 1024 * 1024 + ) { + if (!Number.isSafeInteger(maxRequestBytes) || maxRequestBytes < 1) { + throw new RangeError('Raw request capture requires a finite positive maxRequestBytes.') + } + } + + hasCaptured(req: Request): boolean { + return this.captured.has(req) + } + + async capture(req: Request, res: Response): Promise { + if (req.body !== undefined || req.readableEnded || req.destroyed) { + throw new RawRequestBodyError(400) + } + if (this.pending >= this.maxPending) throw new RawRequestBodyError(503) + if ( + req.headers['content-encoding'] !== undefined && + req.headers['content-encoding'] !== 'identity' + ) { + throw new RawRequestBodyError(415) + } + const declared = req.headers['content-length'] + if ( + declared !== undefined && + (typeof declared !== 'string' || + !/^\d+$/.test(declared) || + !Number.isSafeInteger(Number(declared))) + ) { + throw new RawRequestBodyError(400) + } + if (declared !== undefined && Number(declared) > this.maxRequestBytes) + throw new RawRequestBodyError(413) + this.pending++ + let length = 0 + let released = false + const release = (): void => { + if (released) return + released = true + this.pending-- + this.bufferedBytes -= length + res.off('finish', release) + res.off('close', release) + } + res.once('finish', release) + res.once('close', release) + try { + await new Promise((resolve, reject) => { + const chunks: Buffer[] = [] + let settled = false + const cleanup = (): void => { + clearTimeout(timer) + req.off('data', data) + req.off('end', end) + req.off('error', failed) + req.off('aborted', failed) + res.off('close', failed) + } + const fail = (status: number): void => { + if (settled) return + settled = true + cleanup() + req.pause() + reject(new RawRequestBodyError(status)) + } + const failed = (): void => fail(400) + const data = (chunk: Buffer): void => { + if (!Buffer.isBuffer(chunk)) return fail(400) + if (length + chunk.length > this.maxRequestBytes) return fail(413) + if (this.bufferedBytes + chunk.length > this.maxBufferedBytes) return fail(503) + length += chunk.length + this.bufferedBytes += chunk.length + chunks.push(chunk) + } + const end = (): void => { + if (declared !== undefined && Number(declared) !== length) return fail(400) + settled = true + cleanup() + req.body = length === 0 ? undefined : Buffer.concat(chunks, length) + this.captured.add(req) + resolve() + } + const timer = setTimeout(() => fail(408), this.timeoutMs) + timer.unref?.() + req.on('data', data) + req.once('end', end) + req.once('error', failed) + req.once('aborted', failed) + res.once('close', failed) + }) + } catch (error) { + release() + throw error + } + } +} diff --git a/packages/middleware/payment-express-middleware/README.md b/packages/middleware/payment-express-middleware/README.md index de066fec9..b8d2be3a5 100644 --- a/packages/middleware/payment-express-middleware/README.md +++ b/packages/middleware/payment-express-middleware/README.md @@ -9,6 +9,19 @@ This protocol is distinct from the newer BRC-121 implementation in `@bsv/402-pay`. Choose one protocol deliberately; their headers and client contracts are not interchangeable. +## BRC-118 multipart payments + +Set `enableMultipart: true` behind `createAuthMiddleware({ wallet, +captureRawBody: true })`, installed before any body parser. Negotiation advertises +`header,multipart` only when exact-byte authentication is active. Small payments +retain `x-bsv-payment`; multipart payments share the same amount, BRC-29, BEEF, +internalization and replay validation. The route receives only the original +payload/media type, with exact bytes in `req.rawBody`. Defaults bound payment JSON +to 4 MiB and the complete body to 7 MiB; configure `maxPaymentBytes` and +`maxPaymentBodyBytes` to match the proxy and auth collector. See the +[BRC-118 guide](../../../docs/guides/brc118-payments.md) for receiver-first rollout, +parser policy, browser headers and uncertain payment recovery. + Version 2.1.8 removes middleware payment-header size ceilings. A header already received from the client is parsed and its payment validated regardless of size. HTTP server, CDN, proxy and WAF configuration own transport budgets; configure diff --git a/packages/middleware/payment-express-middleware/package.json b/packages/middleware/payment-express-middleware/package.json index 9728c3d0d..3043a6ec6 100644 --- a/packages/middleware/payment-express-middleware/package.json +++ b/packages/middleware/payment-express-middleware/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/payment-express-middleware", - "version": "2.1.9", + "version": "2.2.0", "sideEffects": false, "engines": { "node": ">=22" @@ -33,11 +33,12 @@ "lint": "oxlint mod.ts src --deny-warnings", "pack:check": "pnpm build && node ../../../scripts/check-package-artifact.mjs . --exports InMemoryPaymentReplayStore,createPaymentMiddleware && node ../../../scripts/check-auth-express-consumers.mjs @bsv/payment-express-middleware", "prepublishOnly": "pnpm build", - "test": "jest --runInBand", + "test": "jest --runInBand && pnpm test:browser", "test:property": "jest --runInBand src/__tests/PaymentReplayStore.property.test.ts", - "test:coverage": "jest --coverage --runInBand", + "test:coverage": "jest --coverage --runInBand && pnpm test:browser", "test:watch": "jest --watch", - "typecheck": "tsc --project tsconfig.typecheck.json" + "typecheck": "tsc --project tsconfig.typecheck.json", + "test:browser": "node test/brc118-browser.mjs" }, "keywords": [ "BSV", @@ -69,21 +70,24 @@ "@types/jest": "^30.0.0", "@types/node": "^26.6.1", "@typescript/native": "npm:typescript@7.0.2", + "esbuild": "0.28.2", "express": "^5.2.1", + "express-rate-limit": "8.6.1", "fast-check": "^4.9.0", "jest": "^30.5.1", "oxlint": "^1.83.0", "ts-jest": "^29.4.12", "ts2md": "^0.2.8", "tsdown": "0.23.0", - "typescript": "npm:@typescript/typescript6@6.0.2" + "typescript": "npm:@typescript/typescript6@6.0.2", + "puppeteer-core": "^25.4.0" }, "bugs": { "url": "https://github.com/bsv-blockchain/ts-stack/issues" }, "homepage": "https://github.com/bsv-blockchain/ts-stack/tree/main/packages/middleware/payment-express-middleware#readme", "peerDependencies": { - "@bsv/sdk": "^2.1.6 || ^3.0.0", + "@bsv/sdk": "^2.9.0 || ^3.0.0", "@types/express": ">=4.17.0 <6", "express": ">=4.18.0 <6" }, diff --git a/packages/middleware/payment-express-middleware/src/__tests/Brc118.integration.test.ts b/packages/middleware/payment-express-middleware/src/__tests/Brc118.integration.test.ts new file mode 100644 index 000000000..a41b496f5 --- /dev/null +++ b/packages/middleware/payment-express-middleware/src/__tests/Brc118.integration.test.ts @@ -0,0 +1,378 @@ +import express from 'express' +import { rateLimit } from 'express-rate-limit' +import { createServer, request as httpRequest, type Server } from 'node:http' +import type { AddressInfo } from 'node:net' +import { + AuthFetch, + Beef, + PrivateKey, + PublicKey, + ProtoWallet, + Script, + Transaction, + P2PKH, + type WalletInterface, + type CreateActionArgs +} from '@bsv/sdk' +import { createAuthMiddleware } from '@bsv/auth-express-middleware' +import { createPaymentMiddleware } from '../index.js' +import type { PaymentRequest } from '../types.js' + +jest.setTimeout(30_000) +const servers: Server[] = [] +async function listen(server: Server): Promise { + servers.push(server) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + return `http://127.0.0.1:${(server.address() as AddressInfo).port}` +} +afterEach(async () => { + jest.restoreAllMocks() + await Promise.all( + servers.splice(0).map(async server => { + server.closeAllConnections() + await new Promise((resolve, reject) => + server.close(error => (error == null ? resolve() : reject(error))) + ) + }) + ) +}) + +async function fixture( + options: { + ancestorBytes?: number + enableMultipart?: boolean + raw?: boolean + proxyBodyLimit?: number + fetchHook?: (url: string, init: RequestInit) => RequestInit + } = {} +) { + const serverWallet = new ProtoWallet(new PrivateKey(23)) as unknown as WalletInterface + const clientWallet = new ProtoWallet(new PrivateKey(24)) as unknown as WalletInterface + const accepted = new Set() + serverWallet.internalizeAction = jest.fn(async args => { + const beef = Beef.fromBinaryStrict(args.tx) + const txid = beef.atomicTxid! + const remittance = args.outputs[0].paymentRemittance! + const key = await serverWallet.getPublicKey({ + protocolID: [2, '3241645161d8'], + keyID: `${remittance.derivationPrefix} ${remittance.derivationSuffix}`, + counterparty: remittance.senderIdentityKey, + forSelf: true + }) + // Derive through the actual wallet, independently of the client's prepared output. + expect(beef.findTxid(txid)!.tx!.outputs[0].lockingScript.toHex()).toBe( + new P2PKH().lock(PublicKey.fromString(key.publicKey).toAddress()).toHex() + ) + if (accepted.has(txid)) return { accepted: true as const, isMerge: true } + accepted.add(txid) + return { accepted: true as const, isMerge: false } + }) + clientWallet.abortAction = jest.fn(async () => ({ aborted: true })) + clientWallet.createAction = jest.fn(async (args: CreateActionArgs) => { + if (args.options?.sendWith !== undefined) + return { + sendWithResults: args.options.sendWith.map(txid => ({ txid, status: 'unproven' as const })) + } + expect(args.options?.noSend).toBe(true) + const source = new Transaction() + source.addOutput({ satoshis: 1000, lockingScript: Script.fromASM('OP_TRUE') }) + if ((options.ancestorBytes ?? 0) > 0) + source.addOutput({ + satoshis: 0, + lockingScript: Script.fromASM(`OP_FALSE OP_RETURN ${'01'.repeat(options.ancestorBytes!)}`) + }) + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + tx.addOutput({ + satoshis: args.outputs![0].satoshis, + lockingScript: Script.fromHex(args.outputs![0].lockingScript) + }) + return { txid: tx.id('hex'), tx: tx.toAtomicBEEF() } + }) + const app = express() + app.use(rateLimit({ windowMs: 60_000, limit: 100 })) + if (options.raw === false) app.use(express.json()) + app.use(createAuthMiddleware({ wallet: serverWallet, captureRawBody: options.raw !== false })) + app.use( + createPaymentMiddleware({ + wallet: serverWallet, + enableMultipart: options.enableMultipart !== false, + calculateRequestPrice: () => 10 + }) + ) + const handler = jest.fn((req: PaymentRequest, res: express.Response) => + res.json({ + method: req.method, + url: req.originalUrl, + mediaType: req.headers['content-type'], + raw: req.rawBody === undefined ? null : Buffer.from(req.rawBody).toString('base64'), + parsed: req.body, + paid: req.payment?.accepted + }) + ) + app.use(handler) + const upstream = await listen(createServer(app)) + const bodyLimit = options.proxyBodyLimit ?? 256 * 1024 + const proxy = createServer({ maxHeaderSize: 6 * 1024 }, (req, res) => { + const chunks: Buffer[] = [] + let size = 0 + if (Number(req.headers['content-length'] ?? 0) > bodyLimit) { + res.writeHead(413, { Connection: 'close' }) + res.end() + return + } + req.on('data', (chunk: Buffer) => { + size += chunk.length + if (size > bodyLimit) { + res.writeHead(413, { Connection: 'close' }) + res.end() + req.pause() + return + } + chunks.push(chunk) + }) + req.on('end', () => { + if (res.writableEnded) return + const forward = httpRequest( + upstream, + { path: req.url, method: req.method, headers: req.headers }, + upstreamResponse => { + res.writeHead(upstreamResponse.statusCode!, upstreamResponse.headers) + upstreamResponse.pipe(res) + } + ) + forward.on('error', () => { + if (!res.headersSent) res.writeHead(502) + res.end() + }) + forward.end(Buffer.concat(chunks)) + }) + }) + const origin = await listen(proxy) + const fetchClient: typeof fetch = async (url, init = {}) => + fetch(url, options.fetchHook?.(String(url), init) ?? init) + const client = new AuthFetch(clientWallet, undefined, undefined, undefined, {}, fetchClient) + jest.spyOn(client as any, 'logPaymentAttempt').mockImplementation(() => {}) + return { client, clientWallet, serverWallet, handler, origin } +} + +describe('BRC-118 through signed HTTP and a 6 KiB-header proxy', () => { + it('keeps absolute request targets on the fixed test upstream', async () => { + let redirectedRequests = 0 + const otherOrigin = await listen( + createServer((_req, res) => { + redirectedRequests++ + res.writeHead(204).end() + }) + ) + const { origin, serverWallet } = await fixture() + const status = await new Promise((resolve, reject) => { + const request = httpRequest(origin, { path: `${otherOrigin}/paid` }, response => { + response.resume() + response.on('end', () => resolve(response.statusCode)) + }) + request.on('error', reject) + request.end() + }) + expect(status).not.toBe(204) + expect(redirectedRequests).toBe(0) + expect(serverWallet.internalizeAction).not.toHaveBeenCalled() + }) + + it.each([ + ['application/json; charset=utf-8', Buffer.from('{ "snow": "雪" }\n')], + ['application/octet-stream', Buffer.from([0, 128, 255, 13, 10, 0])], + ['text/plain', Buffer.alloc(0)], + [ + 'multipart/form-data; boundary=inner', + Buffer.from( + '--inner\r\nContent-Disposition: form-data; name="upload"; filename="file.bin"\r\nContent-Type: application/octet-stream\r\n\r\n\u0000\u00ff\r\n--inner--\r\n' + ) + ] + ])('preserves original %s payload through a large payment', async (contentType, body) => { + const { client, clientWallet, serverWallet, handler, origin } = await fixture({ + ancestorBytes: 12_000 + }) + const response = await client.fetch(`${origin}/paid?q=retained`, { + method: 'POST', + headers: { 'content-type': contentType }, + body, + paymentRetryAttempts: 1 + }) + const result = await response.json() + expect({ status: response.status, result }).toMatchObject({ + status: 200, + result: { paid: true } + }) + expect(result).toMatchObject({ + method: 'POST', + url: '/paid?q=retained', + mediaType: contentType, + raw: body.toString('base64'), + paid: true + }) + expect(handler).toHaveBeenCalledTimes(1) + expect(serverWallet.internalizeAction).toHaveBeenCalledTimes(1) + expect(clientWallet.createAction).toHaveBeenCalledTimes(2) + }) + + it('keeps header transport compatible with a server whose raw multipart path is disabled', async () => { + const { client, origin, handler } = await fixture({ raw: false }) + expect((await client.fetch(`${origin}/paid`, { paymentRetryAttempts: 1 })).status).toBe(200) + expect(handler).toHaveBeenCalledTimes(1) + }) + + it('refuses a large payment for a header-only server and releases the reservation', async () => { + const { client, origin, clientWallet, handler, serverWallet } = await fixture({ + enableMultipart: false, + ancestorBytes: 12_000 + }) + await expect( + client.fetch(`${origin}/paid`, { method: 'POST', paymentRetryAttempts: 1 }) + ).rejects.toMatchObject({ code: 'ERR_PAYMENT_TRANSPORT', payment: { aborted: true } }) + expect(clientWallet.createAction).toHaveBeenCalledTimes(1) + expect(handler).not.toHaveBeenCalled() + expect(serverWallet.internalizeAction).not.toHaveBeenCalled() + }) + + it('treats a real unsigned proxy body refusal as terminal without a second spend', async () => { + const { client, origin, clientWallet, serverWallet, handler } = await fixture({ + ancestorBytes: 12_000, + proxyBodyLimit: 4000 + }) + await expect(client.fetch(`${origin}/paid`, { method: 'POST' })).rejects.toMatchObject({ + code: 'ERR_PAYMENT_SIZE', + httpStatus: 413, + authenticated: false, + payment: { state: 'submitted' } + }) + expect(clientWallet.createAction).toHaveBeenCalledTimes(2) + expect(clientWallet.abortAction).not.toHaveBeenCalled() + expect(serverWallet.internalizeAction).not.toHaveBeenCalled() + expect(handler).not.toHaveBeenCalled() + }) + + it('reproduces a real proxy header refusal below the selection threshold without retrying the spend', async () => { + const { client, origin, clientWallet, serverWallet, handler } = await fixture({ + ancestorBytes: 5000 + }) + await expect(client.fetch(`${origin}/paid`, { method: 'POST' })).rejects.toMatchObject({ + code: 'ERR_PAYMENT_SIZE', + httpStatus: 431, + authenticated: false + }) + expect(clientWallet.createAction).toHaveBeenCalledTimes(2) + expect(clientWallet.abortAction).not.toHaveBeenCalled() + expect(serverWallet.internalizeAction).not.toHaveBeenCalled() + expect(handler).not.toHaveBeenCalled() + }) + + it.each(['boundary', 'payment', 'payload', 'signature'])( + 'rejects modified %s before payment or route work', + async change => { + const { client, origin, handler, serverWallet, clientWallet } = await fixture({ + ancestorBytes: 12_000, + fetchHook: (_url, init) => { + const headers = { ...init.headers } as Record + if (!headers['content-type']?.startsWith('multipart/form-data')) return init + if (change === 'boundary') headers['content-type'] += 'changed' + if (change === 'signature') headers['x-bsv-auth-signature'] = '00' + let body = init.body + if (change === 'payment' || change === 'payload') { + const bytes = Buffer.from(body as Uint8Array) + const target = bytes.indexOf(change === 'payment' ? 'transaction' : 'original') + bytes[target] ^= 1 + body = bytes + } + return { ...init, headers, body } + } + }) + await expect( + client.fetch(`${origin}/paid`, { + method: 'POST', + headers: { 'content-type': 'text/plain' }, + body: 'original', + paymentRetryAttempts: 1 + }) + ).rejects.toThrow() + expect(serverWallet.internalizeAction).not.toHaveBeenCalled() + expect(handler).not.toHaveBeenCalled() + expect(clientWallet.createAction).toHaveBeenCalledTimes(2) + } + ) + it('rejects a newly authenticated replay of the same multipart payment', async () => { + let captured: { body: Uint8Array; contentType: string } | undefined + const { client, origin, handler, serverWallet, clientWallet } = await fixture({ + ancestorBytes: 12_000, + fetchHook: (_url, init) => { + const contentType = (init.headers as Record)['content-type'] + if (contentType?.startsWith('multipart/form-data')) + captured = { body: new Uint8Array(init.body as Uint8Array), contentType } + return init + } + }) + expect((await client.fetch(`${origin}/paid`, { method: 'POST' })).status).toBe(200) + expect(captured).toBeDefined() + const replay = await client.fetch(`${origin}/paid`, { + method: 'POST', + body: captured!.body, + headers: { 'content-type': captured!.contentType } + }) + expect(replay.status).toBe(409) + expect(await replay.json()).toMatchObject({ code: 'ERR_PAYMENT_REPLAYED' }) + expect(handler).toHaveBeenCalledTimes(1) + expect(serverWallet.internalizeAction).toHaveBeenCalledTimes(2) + expect(clientWallet.createAction).toHaveBeenCalledTimes(2) + }) + + it.each([false, true])( + 'does not rerun the route or pay again after a wallet failure (accepted before error=%s)', + async acceptedFirst => { + const { client, origin, handler, serverWallet, clientWallet } = await fixture({ + ancestorBytes: 12_000 + }) + const original = serverWallet.internalizeAction + serverWallet.internalizeAction = jest.fn(async args => { + if (acceptedFirst) await original(args) + throw new Error('private wallet provider context') + }) + const response = await client.fetch(`${origin}/paid`, { method: 'POST' }) + expect(response.ok).toBe(false) + expect(response.status).toBe(400) + const body = await response.text() + expect(body).toContain('ERR_PAYMENT_FAILED') + expect(body).not.toContain('private wallet provider context') + expect(handler).not.toHaveBeenCalled() + expect(serverWallet.internalizeAction).toHaveBeenCalledTimes(1) + expect(clientWallet.createAction).toHaveBeenCalledTimes(2) + expect(clientWallet.abortAction).not.toHaveBeenCalled() + } + ) + + it('rejects simultaneous signed header and multipart payment sources before wallet work', async () => { + let captured: { body: Uint8Array; contentType: string } | undefined + const { client, origin, handler, serverWallet, clientWallet } = await fixture({ + ancestorBytes: 12_000, + fetchHook: (_url, init) => { + const contentType = (init.headers as Record)['content-type'] + if (contentType?.startsWith('multipart/form-data')) + captured = { body: new Uint8Array(init.body as Uint8Array), contentType } + return init + } + }) + expect((await client.fetch(`${origin}/paid`, { method: 'POST' })).status).toBe(200) + const response = await client.fetch(`${origin}/paid`, { + method: 'POST', + body: captured!.body, + headers: { 'content-type': captured!.contentType, 'x-bsv-payment': '{}' } + }) + expect(response.status).toBe(400) + expect(handler).toHaveBeenCalledTimes(1) + expect(serverWallet.internalizeAction).toHaveBeenCalledTimes(1) + expect(clientWallet.createAction).toHaveBeenCalledTimes(2) + }) +}) diff --git a/packages/middleware/payment-express-middleware/src/__tests/MultipartPayment.test.ts b/packages/middleware/payment-express-middleware/src/__tests/MultipartPayment.test.ts new file mode 100644 index 000000000..c385823ed --- /dev/null +++ b/packages/middleware/payment-express-middleware/src/__tests/MultipartPayment.test.ts @@ -0,0 +1,166 @@ +import { buildMultipartPayment } from '@bsv/sdk/auth/utils/paymentTransport' +import { parseMultipartPayment } from '../multipartPayment.js' +import { readFileSync } from 'node:fs' +import { resolve } from 'node:path' + +const boundary = '----BsvPayment0011223344556677' +const payment = '{"derivationPrefix":"AQ==","derivationSuffix":"Ag==","transaction":"AQID"}' +const type = `multipart/form-data; boundary=${boundary}` +function part(name: string, body: string | Buffer, contentType = 'application/json'): Buffer { + return Buffer.concat([ + Buffer.from( + `--${boundary}\r\nContent-Disposition: form-data; name="${name}"\r\nContent-Type: ${contentType}\r\n\r\n` + ), + Buffer.from(body), + Buffer.from('\r\n') + ]) +} +function wire(...parts: Buffer[]): Buffer { + return Buffer.concat([...parts, Buffer.from(`--${boundary}--\r\n`)]) +} +function parse(bytes: Uint8Array, contentType = type) { + return parseMultipartPayment(bytes, contentType, 10_000, 1000) +} + +describe('authenticated BRC-118 extraction', () => { + const vectors = JSON.parse( + readFileSync( + resolve(process.cwd(), '../../../conformance/vectors/payments/brc118.json'), + 'utf8' + ) + ).vectors as Array<{ + id: string + input: { + payment_json: string + payload_base64: string | null + payload_content_type: string | null + } + expected: { body_base64: string; content_type: string } + }> + it.each(vectors)('reads independent Python wire fixture $id', vector => { + const parsed = parse( + Buffer.from(vector.expected.body_base64, 'base64'), + vector.expected.content_type + ) + expect(parsed.paymentJSON).toBe(vector.input.payment_json) + expect(parsed.body === undefined ? null : Buffer.from(parsed.body).toString('base64')).toBe( + vector.input.payload_base64 + ) + expect(parsed.contentType ?? null).toBe(vector.input.payload_content_type) + }) + it.each([ + ['application/json; charset=utf-8', Buffer.from('{ "snow": "雪" }\n')], + ['application/octet-stream', Buffer.from([0, 255, 128, 13, 10, 0])], + ['text/plain', Buffer.from('')], + ['multipart/form-data; boundary=inner', Buffer.from('--inner\r\nunchanged\r\n--inner--')] + ])('preserves bytes and media type for %s', (contentType, body) => { + const built = buildMultipartPayment(payment, { bytes: body, contentType }, 10_000, boundary) + expect(Buffer.from(built.body)).toEqual( + wire(part('x-bsv-payment', payment), part('body', body, contentType)) + ) + const parsed = parse(built.body) + expect(parsed.paymentJSON).toBe(payment) + expect(parsed.contentType).toBe(contentType) + expect(Buffer.from(parsed.body!)).toEqual(body) + expect(parsed.body!.buffer.byteLength).toBe(body.length) + }) + + it('accepts payment-only, quoted boundaries, and either part order', () => { + expect( + parse(wire(part('x-bsv-payment', payment)), `multipart/form-data; boundary="${boundary}"`) + ).toEqual({ paymentJSON: payment, body: undefined, contentType: undefined }) + expect( + Buffer.from( + parse(wire(part('body', 'payload', 'text/plain'), part('x-bsv-payment', payment))).body! + ) + ).toEqual(Buffer.from('payload')) + }) + + it.each([ + wire(), + wire(part('body', 'only')), + wire(part('x-bsv-payment', payment), part('x-bsv-payment', payment)), + wire(part('x-bsv-payment', payment), part('body', ''), part('body', '')), + wire(part('unexpected', payment)), + wire(part('x-bsv-payment', payment, 'text/plain')), + wire(part('x-bsv-payment', payment)).subarray(0, -5), + Buffer.concat([wire(part('x-bsv-payment', payment)), Buffer.from('epilogue')]), + Buffer.from( + wire(part('x-bsv-payment', payment)) + .toString() + .replace('Content-Type:', 'Content-Transfer-Encoding: base64\r\nContent-Type:') + ), + Buffer.from( + wire(part('x-bsv-payment', payment)) + .toString() + .replace('Content-Type:', 'Content-Type: application/json\r\nContent-Type:') + ), + Buffer.from(wire(part('x-bsv-payment', payment)).toString().replaceAll('\r\n', '\n')), + wire(part('x-bsv-payment', Buffer.from([255, 255]))) + ])('rejects malformed or ambiguous framing %#', bytes => { + expect(() => parse(bytes)).toThrow() + }) + + it.each([ + 'multipart/form-data', + `${type}; boundary=second`, + 'multipart/form-data; boundary="a b"', + `${type}\r\nx: y` + ])('rejects invalid Content-Type %j', contentType => { + expect(() => parse(wire(part('x-bsv-payment', payment)), contentType)).toThrow() + }) + + it.each([null, 'not raw bytes', [], { length: 0, buffer: new ArrayBuffer(0) }])( + 'rejects a tampered raw-body type %# before reading framing', + value => { + expect(() => parse(value as unknown as Uint8Array)).toThrow('Malformed multipart payment') + } + ) + + it.each([ + 'Content-Type', + ': application/json', + 'Content Type: application/json', + `Content-Type:${' '.repeat(2050)}` + ])('rejects malformed or oversized header lines %#', header => { + const bytes = Buffer.from( + wire(part('x-bsv-payment', payment)) + .toString() + .replace('Content-Type: application/json', header) + ) + expect(() => parse(bytes)).toThrow('Malformed multipart payment') + }) + + it('measures the raw byte span independently of a shadowed array length', () => { + const bytes = new Uint8Array(wire(part('x-bsv-payment', payment))) + Object.defineProperty(bytes, 'length', { value: 0 }) + expect(() => parseMultipartPayment(bytes, type, bytes.byteLength - 1, 1000)).toThrow('limit') + }) + + it('bounds body, payment, and part headers', () => { + const bytes = wire(part('x-bsv-payment', payment)) + expect(() => parseMultipartPayment(bytes, type, bytes.length - 1, 1000)).toThrow('limit') + expect(() => parseMultipartPayment(bytes, type, 10_000, payment.length - 1)).toThrow('limit') + expect(() => + parse(wire(part('x-bsv-payment', payment, `application/json;${'x'.repeat(2100)}`))) + ).toThrow() + }) + + it('does not truncate a binary payload at a boundary prefix', () => { + const body = Buffer.from(`abc\r\n--${boundary}not-a-delimiter\r\nend`) + expect( + Buffer.from( + parse(wire(part('x-bsv-payment', payment), part('body', body, 'application/octet-stream'))) + .body! + ) + ).toEqual(body) + expect(() => + buildMultipartPayment( + payment, + { bytes: body, contentType: 'application/octet-stream' }, + 10_000, + boundary + ) + ).toThrow('collides') + }) +}) diff --git a/packages/middleware/payment-express-middleware/src/index.ts b/packages/middleware/payment-express-middleware/src/index.ts index c66270daf..b76770ccc 100644 --- a/packages/middleware/payment-express-middleware/src/index.ts +++ b/packages/middleware/payment-express-middleware/src/index.ts @@ -1,6 +1,9 @@ +import { isMultipartPaymentType, PaymentTransportError } from '@bsv/sdk/auth/utils/paymentTransport' +import { decodePaymentPayload } from '@bsv/sdk/auth/utils/decodePaymentPayload' +import { parseMultipartPayment, MissingMultipartPayment } from './multipartPayment.js' import { toArray, toBase64 } from '@bsv/sdk/primitives/utils' import { Beef, createNonce, PublicKey, verifyNonce, type AtomicBEEF } from '@bsv/sdk' -import type { RequestHandler, Response } from 'express' +import type { NextFunction, RequestHandler, Response } from 'express' import type { BSVPayment, PaymentMiddlewareOptions, @@ -42,17 +45,18 @@ function isPositiveSafeInteger(value: number): boolean { } function isCanonicalBase64(value: string): boolean { - if ( - value.length === 0 || - !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value) - ) { - return false - } - try { - return toBase64(toArray(value, 'base64')) === value - } catch { + if (value.length === 0 || value.length % 4 !== 0 || !/^[A-Za-z0-9+/]+={0,2}$/.test(value)) { return false } + // Check pad bits without decoding or using a repeated-group regex, whose + // engine stack can overflow on the large BEEFs multipart was designed for. + let padding = 0 + if (value.endsWith('==')) padding = 2 + else if (value.endsWith('=')) padding = 1 + const last = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/'.indexOf( + value[value.length - padding - 1] + ) + return padding === 0 || (padding === 1 ? (last & 3) === 0 : (last & 15) === 0) } function isCompressedPublicKey(value: string): boolean { @@ -184,7 +188,8 @@ async function issuePaymentChallenge( wallet: PaymentMiddlewareOptions['wallet'], res: Response, requestPrice: number, - logger: PaymentMiddlewareOptions['logger'] + logger: PaymentMiddlewareOptions['logger'], + multipart = false ): Promise { try { const derivationPrefix = await createNonce(wallet) @@ -193,7 +198,8 @@ async function issuePaymentChallenge( .set({ 'x-bsv-payment-version': PAYMENT_VERSION, 'x-bsv-payment-satoshis-required': String(requestPrice), - 'x-bsv-payment-derivation-prefix': derivationPrefix + 'x-bsv-payment-derivation-prefix': derivationPrefix, + 'x-bsv-payment-transports': multipart ? 'header,multipart' : 'header' }) .json({ status: 'error', @@ -207,6 +213,144 @@ async function issuePaymentChallenge( } } +interface PaymentInputLimits { + enableMultipart: boolean + maxPaymentBodyBytes: number + maxPaymentBytes: number +} + +function restoreApplicationPayload( + req: PaymentRequest, + parsed: ReturnType +): void { + req.rawBody = parsed.body + req.body = decodePaymentPayload(parsed.body, parsed.contentType) + delete req.headers['content-type'] + delete req.headers['content-length'] + delete req.headers['transfer-encoding'] + if (parsed.contentType !== undefined) req.headers['content-type'] = parsed.contentType + if (parsed.body !== undefined) req.headers['content-length'] = String(parsed.body.length) +} + +function extractPaymentInput( + req: PaymentRequest, + res: Response, + multipart: boolean, + limits: PaymentInputLimits +): { rawPayment: string | null | undefined } | undefined { + const { enableMultipart, maxPaymentBodyBytes, maxPaymentBytes } = limits + const contentType = req.headers['content-type'] + const rawPayment = paymentHeader(req) + const headerInput = { rawPayment } + if (!enableMultipart || typeof contentType !== 'string' || !isMultipartPaymentType(contentType)) + return headerInput + if (!multipart || rawPayment !== undefined || !(req.body instanceof Uint8Array)) { + sendError( + res, + 400, + 'ERR_MALFORMED_PAYMENT', + 'Multipart payments require raw authentication and exactly one payment source.' + ) + return undefined + } + try { + const parsed = parseMultipartPayment( + req.body, + contentType, + maxPaymentBodyBytes, + maxPaymentBytes + ) + restoreApplicationPayload(req, parsed) + return { rawPayment: parsed.paymentJSON } + } catch (error) { + if (error instanceof MissingMultipartPayment) return headerInput + const status = + error instanceof PaymentTransportError && error.code === 'ERR_PAYMENT_SIZE' ? 413 : 400 + sendError( + res, + status, + 'ERR_MALFORMED_PAYMENT', + 'The multipart payment is malformed or exceeds its limit.' + ) + return undefined + } +} + +async function settlePayment( + paymentRequest: PaymentRequest, + res: Response, + next: NextFunction, + parsed: ParsedPayment, + identityKey: string, + options: Required> & + Pick +): Promise { + const { wallet, replayStore, logger } = options + try { + const result: unknown = await wallet.internalizeAction({ + tx: parsed.transaction, + outputs: [ + { + paymentRemittance: { + derivationPrefix: parsed.payment.derivationPrefix, + derivationSuffix: parsed.payment.derivationSuffix, + senderIdentityKey: identityKey + }, + outputIndex: 0, + protocol: 'wallet payment' + } + ], + description: 'Payment for request' + }) + + if (!isNewlyAcceptedInternalization(result)) { + sendError(res, 409, 'ERR_PAYMENT_REPLAYED', 'This payment was not newly accepted.') + return + } + + // The wallet is the authority that validates the remittance and records + // whether it was newly accepted. Claim only after that validation so an + // attacker cannot poison a transaction ID by pairing a public BEEF with + // invalid derivation material. A buggy wallet that accepts a duplicate is + // still contained by the independent atomic replay store. + let claimed: boolean + try { + const claimResult: unknown = await replayStore.claim(parsed.transactionId) + if (typeof claimResult !== 'boolean') { + throw new TypeError('The replay store returned an invalid claim result.') + } + claimed = claimResult + } catch (error) { + emitLog(logger, 'error', 'Payment replay claim failed.', safeErrorContext(error)) + sendError( + res, + 503, + 'ERR_PAYMENT_UNAVAILABLE', + 'Payment processing is temporarily unavailable.' + ) + return + } + if (!claimed) { + sendError(res, 409, 'ERR_PAYMENT_REPLAYED', 'This payment was already used.') + return + } + + paymentRequest.payment = { + satoshisPaid: parsed.satoshis, + accepted: true, + tx: toBase64(parsed.transaction), + txid: parsed.transactionId + } + res.set({ + 'x-bsv-payment-satoshis-paid': String(parsed.satoshis) + }) + next() + } catch (error) { + emitLog(logger, 'warn', 'Payment internalization failed.', safeErrorContext(error)) + sendError(res, 400, 'ERR_PAYMENT_FAILED', 'The payment could not be accepted.') + } +} + /** * Creates middleware that enforces a BRC-29 wallet payment after BRC-103 auth. */ @@ -219,6 +363,9 @@ export function createPaymentMiddleware(options: PaymentMiddlewareOptions): Requ calculateRequestPrice = () => 100, wallet, replayStore = new InMemoryPaymentReplayStore(), + enableMultipart = false, + maxPaymentBodyBytes = 7 * 1024 * 1024, + maxPaymentBytes = 4 * 1024 * 1024, logger } = options @@ -238,6 +385,12 @@ export function createPaymentMiddleware(options: PaymentMiddlewareOptions): Requ if (!isPaymentLogger(logger)) { throw new TypeError('logger error and warn properties must be functions when provided.') } + if (typeof enableMultipart !== 'boolean') + throw new TypeError('enableMultipart must be a boolean.') + for (const value of [maxPaymentBodyBytes, maxPaymentBytes]) { + if (!Number.isSafeInteger(value) || value < 1 || value > 16 * 1024 * 1024) + throw new RangeError('Multipart payment limits must be integers from 1 through 16777216.') + } return async (req, res, next): Promise => { const paymentRequest: PaymentRequest = req @@ -252,6 +405,15 @@ export function createPaymentMiddleware(options: PaymentMiddlewareOptions): Requ return } + const multipart = enableMultipart && paymentRequest.auth?.supportsMultipart === true + const input = extractPaymentInput(paymentRequest, res, multipart, { + enableMultipart, + maxPaymentBodyBytes, + maxPaymentBytes + }) + if (input === undefined) return + const { rawPayment } = input + let requestPrice: number try { requestPrice = await calculateRequestPrice(paymentRequest) @@ -277,9 +439,8 @@ export function createPaymentMiddleware(options: PaymentMiddlewareOptions): Requ return } - const rawPayment = paymentHeader(paymentRequest) if (rawPayment === undefined) { - await issuePaymentChallenge(wallet, res, requestPrice, logger) + await issuePaymentChallenge(wallet, res, requestPrice, logger, multipart) return } @@ -326,68 +487,10 @@ export function createPaymentMiddleware(options: PaymentMiddlewareOptions): Requ return } - try { - const result: unknown = await wallet.internalizeAction({ - tx: parsed.transaction, - outputs: [ - { - paymentRemittance: { - derivationPrefix: payment.derivationPrefix, - derivationSuffix: payment.derivationSuffix, - senderIdentityKey: identityKey - }, - outputIndex: 0, - protocol: 'wallet payment' - } - ], - description: 'Payment for request' - }) - - if (!isNewlyAcceptedInternalization(result)) { - sendError(res, 409, 'ERR_PAYMENT_REPLAYED', 'This payment was not newly accepted.') - return - } - - // The wallet is the authority that validates the remittance and records - // whether it was newly accepted. Claim only after that validation so an - // attacker cannot poison a transaction ID by pairing a public BEEF with - // invalid derivation material. A buggy wallet that accepts a duplicate is - // still contained by the independent atomic replay store. - let claimed: boolean - try { - const claimResult: unknown = await replayStore.claim(parsed.transactionId) - if (typeof claimResult !== 'boolean') { - throw new TypeError('The replay store returned an invalid claim result.') - } - claimed = claimResult - } catch (error) { - emitLog(logger, 'error', 'Payment replay claim failed.', safeErrorContext(error)) - sendError( - res, - 503, - 'ERR_PAYMENT_UNAVAILABLE', - 'Payment processing is temporarily unavailable.' - ) - return - } - if (!claimed) { - sendError(res, 409, 'ERR_PAYMENT_REPLAYED', 'This payment was already used.') - return - } - - paymentRequest.payment = { - satoshisPaid: parsed.satoshis, - accepted: true, - tx: toBase64(parsed.transaction), - txid: parsed.transactionId - } - res.set({ - 'x-bsv-payment-satoshis-paid': String(parsed.satoshis) - }) - next() - } catch (error) { - emitLog(logger, 'warn', 'Payment internalization failed.', safeErrorContext(error)) - sendError(res, 400, 'ERR_PAYMENT_FAILED', 'The payment could not be accepted.') - } + await settlePayment(paymentRequest, res, next, parsed, identityKey, { + wallet, + replayStore, + logger + }) } } diff --git a/packages/middleware/payment-express-middleware/src/multipartPayment.ts b/packages/middleware/payment-express-middleware/src/multipartPayment.ts new file mode 100644 index 000000000..2a63f9aab --- /dev/null +++ b/packages/middleware/payment-express-middleware/src/multipartPayment.ts @@ -0,0 +1,143 @@ +import { + paymentBoundary, + paymentPayloadContentType, + PaymentTransportError +} from '@bsv/sdk/auth/utils/paymentTransport' +import { toUTF8Strict } from '@bsv/sdk/primitives/utils' + +export interface ParsedMultipartPayment { + paymentJSON: string + body?: Uint8Array + contentType?: string +} + +/** An ordinary, unpaid multipart application request needs the normal 402 challenge. */ +export class MissingMultipartPayment extends Error {} + +function malformed(): never { + throw new PaymentTransportError('ERR_PAYMENT_TRANSPORT', 'Malformed multipart payment.') +} + +interface Part { + name: string + contentType?: string + bytes: Uint8Array + next: number + final: boolean +} + +function partHeaders( + source: Buffer, + position: number +): { headers: Map; start: number } { + // Bound the header search itself, before reading any attacker-sized field. + const window = source.subarray(position, Math.min(position + 2048 + 4, source.length)) + const length = window.indexOf('\r\n\r\n') + if (length < 0 || length > 2048) malformed() + const bytes = window.subarray(0, length) + if (bytes.some(byte => byte > 126 || (byte < 32 && byte !== 13 && byte !== 10))) malformed() + const headers = new Map() + for (const line of bytes.toString('ascii').split('\r\n')) { + const colon = line.indexOf(':') + const name = line.slice(0, colon).toLowerCase() + if (colon < 1 || !/^[a-z-]+$/.test(name)) malformed() + if (headers.has(name) || (name !== 'content-disposition' && name !== 'content-type')) + malformed() + headers.set(name, line.slice(colon + 1).trim()) + } + return { headers, start: position + length + 4 } +} + +function partEnd(source: Buffer, delimiter: Buffer, start: number): number { + let end = source.indexOf(delimiter, start) + while (end !== -1) { + const suffix = source + .subarray(end + delimiter.length, end + delimiter.length + 2) + .toString('ascii') + if (suffix === '\r\n' || suffix === '--') return end + end = source.indexOf(delimiter, end + delimiter.length) + } + return malformed() +} + +function readPart(source: Buffer, delimiter: Buffer, position: number): Part { + const { headers, start } = partHeaders(source, position) + const disposition = + /^form-data;\s*name="([^"\r\n]{1,128})"(?:;\s*filename="[^"\r\n]{0,256}")?$/i.exec( + headers.get('content-disposition') ?? '' + ) + if (disposition === null) malformed() + const contentType = headers.get('content-type') + if (contentType !== undefined) paymentPayloadContentType(contentType) + const end = partEnd(source, delimiter, start) + const suffix = end + delimiter.length + const final = source.subarray(suffix, suffix + 2).toString('ascii') === '--' + const next = suffix + 2 + if (final) { + const epilogue = source.subarray(next).toString('ascii') + if (epilogue !== '' && epilogue !== '\r\n') malformed() + } + return { name: disposition[1], contentType, bytes: source.subarray(start, end), next, final } +} + +interface CollectedParts { + paymentJSON?: string + body?: Uint8Array + contentType?: string + invalid: boolean +} + +function collectPart(result: CollectedParts, part: Part, maxPaymentBytes: number): void { + if (part.name === 'x-bsv-payment') { + if ( + part.contentType === undefined || + !/^application\/json(?:;\s*charset=utf-8)?$/i.test(part.contentType) + ) + malformed() + if (part.bytes.length > maxPaymentBytes) + throw new PaymentTransportError( + 'ERR_PAYMENT_SIZE', + 'Multipart payment JSON exceeds its limit.' + ) + result.paymentJSON = toUTF8Strict(part.bytes) + } else if (part.name === 'body') { + if (part.contentType === undefined) result.invalid = true + // Own inner bytes so downstream views cannot retain or expose the outer payment part. + result.body = new Uint8Array(part.bytes) + result.contentType = part.contentType + } else result.invalid = true +} + +/** A bounded two-part RFC 7578 profile. Run only after authenticating the complete raw request. */ +export function parseMultipartPayment( + bytes: Uint8Array, + contentType: string, + maxBodyBytes: number, + maxPaymentBytes: number +): ParsedMultipartPayment { + if (!(bytes instanceof Uint8Array)) malformed() + if (bytes.byteLength > maxBodyBytes) + throw new PaymentTransportError('ERR_PAYMENT_SIZE', 'Multipart payment body exceeds its limit.') + const boundary = paymentBoundary(contentType) + const source = Buffer.from(bytes.buffer, bytes.byteOffset, bytes.byteLength) + const delimiter = Buffer.from(`\r\n--${boundary}`) + const opening = Buffer.from(`--${boundary}\r\n`) + if (source.equals(Buffer.from(`--${boundary}--\r\n`))) throw new MissingMultipartPayment() + if (!source.subarray(0, opening.length).equals(opening)) malformed() + let position = opening.length + let parts = 0 + const result: CollectedParts = { invalid: false } + const seen = new Set() + while (true) { + if (++parts > 128) malformed() + const part = readPart(source, delimiter, position) + if (seen.has(part.name)) result.invalid = true + seen.add(part.name) + collectPart(result, part, maxPaymentBytes) + if (part.final) break + position = part.next + } + if (result.paymentJSON === undefined) throw new MissingMultipartPayment() + if (result.invalid || parts > 2) malformed() + return { paymentJSON: result.paymentJSON, body: result.body, contentType: result.contentType } +} diff --git a/packages/middleware/payment-express-middleware/src/types.ts b/packages/middleware/payment-express-middleware/src/types.ts index 67daffba1..8f1ec1819 100644 --- a/packages/middleware/payment-express-middleware/src/types.ts +++ b/packages/middleware/payment-express-middleware/src/types.ts @@ -11,7 +11,9 @@ export interface PaymentReceipt { export interface PaymentRequest extends Request { auth?: { identityKey?: unknown + supportsMultipart?: boolean } + rawBody?: Uint8Array payment?: PaymentReceipt } @@ -37,6 +39,12 @@ export interface PaymentMiddlewareOptions { * header budgets on the HTTP server, CDN, proxy or WAF before middleware. */ maxPaymentHeaderBytes?: number + /** Advertise multipart only when raw-byte auth support is present on the request. */ + enableMultipart?: boolean + /** Complete multipart request size, default 7 MiB. */ + maxPaymentBodyBytes?: number + /** Serialized payment part size, default 4 MiB. */ + maxPaymentBytes?: number logger?: PaymentLogger } diff --git a/packages/middleware/payment-express-middleware/test/brc118-browser.mjs b/packages/middleware/payment-express-middleware/test/brc118-browser.mjs new file mode 100644 index 000000000..0cbb4def8 --- /dev/null +++ b/packages/middleware/payment-express-middleware/test/brc118-browser.mjs @@ -0,0 +1,224 @@ +import assert from 'node:assert/strict' +import { createServer } from 'node:http' +import { access, readFile } from 'node:fs/promises' +import { fileURLToPath, pathToFileURL } from 'node:url' +import express from 'express' +import { build } from 'esbuild' +import puppeteer from 'puppeteer-core' +import { Beef, PrivateKey, ProtoWallet, PublicKey, P2PKH } from '@bsv/sdk' +import { createAuthMiddleware } from '@bsv/auth-express-middleware' +import { createPaymentMiddleware } from '../dist/mod.mjs' + +const candidates = [ + process.env.CHROME_BIN, + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/usr/bin/google-chrome', + '/usr/bin/chromium' +].filter(Boolean) +let executablePath +for (const candidate of candidates) { + try { + await access(candidate) + executablePath = candidate + break + } catch { + /* next installed browser */ + } +} +assert.ok( + executablePath, + 'Chrome/Chromium or CHROME_BIN is required; the BRC-118 browser gate cannot be skipped.' +) +const bundleOptions = { + entryPoints: [fileURLToPath(new URL('./brc118-client.mjs', import.meta.url))], + bundle: true, + write: false, + platform: 'browser', + format: 'esm', + target: 'es2022' +} +const bundle = await build(bundleOptions) +let legacyBundle +const servers = [] +async function listen(server) { + servers.push(server) + await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) + return `http://127.0.0.1:${server.address().port}` +} +async function* receivers(cases) { + for (const item of cases) yield receiver(item).then(target => ({ item, target })) +} + +async function receiver({ multipart = true, exposeTransport = true } = {}) { + const app = express() + const wallet = new ProtoWallet(new PrivateKey(23)) + let accepted = 0 + let handled = 0 + let preflights = 0 + wallet.internalizeAction = async args => { + const beef = Beef.fromBinaryStrict(args.tx) + const remittance = args.outputs[0].paymentRemittance + const key = await wallet.getPublicKey({ + protocolID: [2, '3241645161d8'], + keyID: `${remittance.derivationPrefix} ${remittance.derivationSuffix}`, + counterparty: remittance.senderIdentityKey, + forSelf: true + }) + const output = beef.findTxid(beef.atomicTxid).tx.outputs[0] + assert.equal( + output.lockingScript.toHex(), + new P2PKH().lock(PublicKey.fromString(key.publicKey).toAddress()).toHex() + ) + assert.equal(output.satoshis, 10) + accepted++ + return { accepted: true, isMerge: false } + } + app.use((req, res, next) => { + res.setHeader('Access-Control-Allow-Origin', '*') + res.setHeader('Access-Control-Allow-Methods', 'POST, OPTIONS') + res.setHeader( + 'Access-Control-Allow-Headers', + req.headers['access-control-request-headers'] ?? '*' + ) + res.setHeader( + 'Access-Control-Expose-Headers', + exposeTransport + ? '*' + : 'x-bsv-auth-version, x-bsv-auth-identity-key, x-bsv-auth-nonce, x-bsv-auth-your-nonce, x-bsv-auth-signature, x-bsv-auth-request-id, x-bsv-payment-version, x-bsv-payment-satoshis-required, x-bsv-payment-derivation-prefix' + ) + if (req.method === 'OPTIONS') { + preflights++ + res.sendStatus(204) + return + } + next() + }) + app.use(createAuthMiddleware({ wallet, captureRawBody: true })) + app.use( + createPaymentMiddleware({ wallet, enableMultipart: multipart, calculateRequestPrice: () => 10 }) + ) + app.use((req, res) => { + handled++ + res.json({ + raw: Array.from(req.rawBody ?? []), + mediaType: req.headers['content-type'], + paid: req.payment.accepted + }) + }) + return { + origin: await listen(createServer(app)), + counts: () => ({ accepted, handled, preflights }) + } +} +let browser +try { + const pageOrigin = await listen( + createServer((req, res) => { + res.setHeader('Content-Type', req.url?.endsWith('.js') ? 'text/javascript' : 'text/html') + if (req.url === '/legacy.js') res.end(legacyBundle?.outputFiles[0].contents) + else if (req.url === '/client.js') res.end(bundle.outputFiles[0].contents) + else + res.end( + 'BRC-118 cross-origin acceptance' + ) + }) + ) + browser = await puppeteer.launch({ executablePath, headless: true, args: ['--no-sandbox'] }) + const page = await browser.newPage() + page.on('pageerror', error => process.stderr.write(`${error}\n`)) + await page.goto(pageOrigin) + await page.waitForFunction('typeof globalThis.pay === "function"') + const cases = [ + { contentType: 'application/octet-stream', body: [0, 255, 128, 13, 10], ancestorBytes: 12000 }, + { + contentType: 'application/json; charset=utf-8', + body: Array.from(Buffer.from('{ "snow": "雪" }\n')), + ancestorBytes: 12000 + }, + { contentType: 'text/plain', body: [], ancestorBytes: 0, multipart: false } + ] + // Reuse the page only after the preceding wallet scenario has settled. + for await (const { item, target } of receivers(cases)) { + const result = await page.evaluate(async args => await globalThis.pay(args), { + ...item, + origin: target.origin + }) + assert.equal(result.status, 200, JSON.stringify(result)) + assert.deepEqual(result.result.raw, item.body) + assert.equal(result.result.mediaType, item.contentType) + assert.equal(result.result.paid, true) + assert.deepEqual([result.prepared, result.submitted, result.aborted], [1, 1, 0]) + assert.ok(result.challenges.includes(item.multipart === false ? 'header' : 'header,multipart')) + assert.equal(target.counts().accepted, 1) + assert.equal(target.counts().handled, 1) + assert.ok(target.counts().preflights > 0, 'Browser did not exercise cross-origin preflight') + console.log( + JSON.stringify({ + browserPayment: item.contentType, + ancestorBytes: item.ancestorBytes, + ...target.counts() + }) + ) + } + const hidden = await receiver({ exposeTransport: false }) + const refusal = await page.evaluate(async args => await globalThis.pay(args), { + origin: hidden.origin, + ancestorBytes: 12000, + contentType: 'text/plain', + body: [] + }) + // Negotiation is signed: hiding it invalidates authentication before any wallet spend. + assert.equal(refusal.code, 'ERR_INVALID_SIGNATURE', JSON.stringify(refusal)) + assert.deepEqual([refusal.prepared, refusal.submitted, refusal.aborted], [0, 0, 0]) + assert.equal(hidden.counts().accepted, 0) + assert.equal(hidden.counts().handled, 0) + console.log( + 'BRC-118 native browser: binary, exact JSON, header-only compatibility and hidden-capability refusal passed.' + ) + // Optional compatibility probe is additive: the complete current-client gate above always runs. + const legacyModule = process.env.BRC118_LEGACY_SDK_MODULE + if (legacyModule !== undefined) { + const manifest = JSON.parse( + await readFile(new URL('../../package.json', pathToFileURL(legacyModule)), 'utf8') + ) + assert.equal(manifest.name, '@bsv/sdk') + assert.equal(manifest.version, '2.8.0') + legacyBundle = await build({ ...bundleOptions, alias: { '@bsv/sdk': legacyModule } }) + await page.addScriptTag({ url: `${pageOrigin}/legacy.js`, type: 'module' }) + for await (const { + item: { multipart }, + target + } of receivers([{ multipart: true }, { multipart: false }])) { + const item = { + origin: target.origin, + ancestorBytes: 0, + contentType: 'text/plain', + body: [108, 101, 103, 97, 99, 121], + legacy: true + } + const result = await page.evaluate(async args => await globalThis.pay(args), item) + assert.equal(result.status, 200, JSON.stringify(result)) + assert.deepEqual(result.result.raw, item.body) + assert.deepEqual([result.prepared, result.submitted, result.aborted], [0, 1, 0]) + assert.equal(target.counts().accepted, 1) + assert.equal(target.counts().handled, 1) + console.log( + JSON.stringify({ + publishedClient: '2.8.0', + receiverMultipart: multipart, + ...target.counts() + }) + ) + } + } +} finally { + if (browser !== undefined) await browser.close() + await Promise.all( + servers.map(async server => { + server.closeAllConnections() + await new Promise((resolve, reject) => + server.close(error => (error ? reject(error) : resolve())) + ) + }) + ) +} diff --git a/packages/middleware/payment-express-middleware/test/brc118-client.mjs b/packages/middleware/payment-express-middleware/test/brc118-client.mjs new file mode 100644 index 000000000..f1a891a30 --- /dev/null +++ b/packages/middleware/payment-express-middleware/test/brc118-client.mjs @@ -0,0 +1,67 @@ +import { AuthFetch, PrivateKey, ProtoWallet, Script, Transaction } from '@bsv/sdk' + +// This code runs in an actual cross-origin browser without Node globals. +globalThis.pay = async ({ origin, ancestorBytes, contentType, body, legacy = false }) => { + const wallet = new ProtoWallet(new PrivateKey(24)) + let prepared = 0 + let submitted = 0 + let aborted = 0 + wallet.abortAction = async () => { + aborted++ + return { aborted: true } + } + wallet.createAction = async args => { + if (args.options?.sendWith !== undefined) { + submitted++ + return { sendWithResults: args.options.sendWith.map(txid => ({ txid, status: 'unproven' })) } + } + if (legacy) submitted++ + else { + prepared++ + if (args.options?.noSend !== true) + throw new Error('Payment was not prepared before submission') + } + const source = new Transaction() + source.addOutput({ satoshis: 1000, lockingScript: Script.fromASM('OP_TRUE') }) + if (ancestorBytes > 0) + source.addOutput({ + satoshis: 0, + lockingScript: Script.fromASM(`OP_FALSE OP_RETURN ${'01'.repeat(ancestorBytes)}`) + }) + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + tx.addOutput({ + satoshis: args.outputs[0].satoshis, + lockingScript: Script.fromHex(args.outputs[0].lockingScript) + }) + return { txid: tx.id('hex'), tx: tx.toAtomicBEEF() } + } + const challenges = [] + const client = new AuthFetch(wallet, undefined, undefined, undefined, {}, async (url, init) => { + const response = await fetch(url, init) + if (response.status === 402) challenges.push(response.headers.get('x-bsv-payment-transports')) + return response + }) + try { + const response = await client.fetch(`${origin}/paid`, { + method: 'POST', + headers: { 'content-type': contentType }, + body: new Uint8Array(body), + paymentRetryAttempts: 1 + }) + return { + status: response.status, + result: await response.json(), + prepared, + submitted, + aborted, + challenges + } + } catch (error) { + return { code: error.code, prepared, submitted, aborted, challenges } + } +} diff --git a/packages/overlays/overlay-discovery-services/CHANGELOG.md b/packages/overlays/overlay-discovery-services/CHANGELOG.md index 6a0eea82f..c6a522599 100644 --- a/packages/overlays/overlay-discovery-services/CHANGELOG.md +++ b/packages/overlays/overlay-discovery-services/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG for `@bsv/overlay-discovery-services` +## 2.2.7 (unreleased) + +- Refresh packed first-party dependency ranges for the next wallet interoperability release; no independent API migration. + All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Table of Contents diff --git a/packages/overlays/overlay-discovery-services/README.md b/packages/overlays/overlay-discovery-services/README.md index 2dd75df27..433957c8a 100644 --- a/packages/overlays/overlay-discovery-services/README.md +++ b/packages/overlays/overlay-discovery-services/README.md @@ -176,3 +176,10 @@ Current TS Stack changes are licensed under the Open BSV License Version 6; see under the Open BSV License Version 4. Redistributors must preserve [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md) and the applicable text in [`LICENSES/`](./LICENSES/). + +## Next dependency release candidate + +This candidate refreshes the packed first-party dependency ranges for the next +wallet interoperability release. It adds no independent API or wire-format +change. Adopt after the new dependency graph is published; current wallet +releases retain their existing published pins. diff --git a/packages/overlays/overlay-express/CHANGELOG.md b/packages/overlays/overlay-express/CHANGELOG.md index 272ae5b90..f677406d1 100644 --- a/packages/overlays/overlay-express/CHANGELOG.md +++ b/packages/overlays/overlay-express/CHANGELOG.md @@ -1,5 +1,9 @@ # CHANGELOG for `@bsv/overlay-express` +## 2.7.4 (unreleased) + +- Refresh packed first-party dependency ranges for the next wallet interoperability release; no independent API migration. + All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## Table of Contents diff --git a/packages/overlays/overlay-express/README.md b/packages/overlays/overlay-express/README.md index 6c5d28da7..f7c7c9b65 100644 --- a/packages/overlays/overlay-express/README.md +++ b/packages/overlays/overlay-express/README.md @@ -433,3 +433,10 @@ under the Open BSV License Version 4. Redistributors must preserve [`LICENSES/`](./LICENSES/). Thank you for being a part of the BSV Blockchain Overlay Express Project. Let's build the future of BSV Blockchain together! + +## Next dependency release candidate + +This candidate refreshes the packed first-party dependency ranges for the next +wallet interoperability release. It adds no independent API or wire-format +change. Adopt after the new dependency graph is published; current wallet +releases retain their existing published pins. diff --git a/packages/overlays/topics/CHANGELOG.md b/packages/overlays/topics/CHANGELOG.md index b5b873fec..6dfefe34d 100644 --- a/packages/overlays/topics/CHANGELOG.md +++ b/packages/overlays/topics/CHANGELOG.md @@ -9,6 +9,11 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +### Dependency update (2.0.0 candidate) + +- Refresh the packed templates range to include its CommonJS interoperability + repair. This dependency repair is independent of the API removals below. + ### Removed (2.0.0 candidate) - Remove `tm_did` / `ls_did`, `DIDTopicManager`, `createDIDLookupService`, diff --git a/packages/overlays/topics/README.md b/packages/overlays/topics/README.md index d0e93eb1b..740680b9e 100644 --- a/packages/overlays/topics/README.md +++ b/packages/overlays/topics/README.md @@ -38,6 +38,10 @@ Requires Node.js 22 or newer. Install `@bsv/sdk` alongside this package to satisfy its peer dependency. The overlay engine, templates, and MongoDB driver are direct runtime dependencies. +The unpublished 1.9.2 candidate refreshes the packed templates range for its +CommonJS compatibility repair. Topic APIs, schemas and admission behavior are +unchanged; no API migration is required. + ## Quick start Register a managed topic on an overlay engine: diff --git a/packages/sdk/CHANGELOG.md b/packages/sdk/CHANGELOG.md index 941fd683d..5bbdaa08d 100644 --- a/packages/sdk/CHANGELOG.md +++ b/packages/sdk/CHANGELOG.md @@ -214,6 +214,24 @@ All notable changes to this project will be documented in this file. The format ## [Unreleased] +### 3.0.0 candidate — prepared BRC-118 payments and recipient interoperability + +- Preserve native body serialization and inferred Content-Type on ordinary HTTP + fallback, including cached unauthenticated peers; retain owned payment bytes. + +- Negotiate bounded multipart payments, preserving original payload bytes and exact + signed boundary parameters while retaining nonempty non-multipart signature preimages. +- Prepare the real payment and request before broadcast; submit once, reuse the + transaction across retries, abort refused reservations where supported, and + retain typed context for cancellation, size refusal and uncertain outcomes. +- Pair zero-length byte request normalization with auth middleware 2.3.0: both + use the BRC-104 `-1` sentinel, including the existing Express raw-parser path. + Empty-byte clients/receivers that used length `0` need a coordinated upgrade; + nonempty request preimages are unchanged. +- Derive the recipient's own BRC-29 child key on settlement receipt. +- Add independent Python wire/preimage vectors, real HTTP/proxy-limit tests and + adversarial payment lifecycle coverage. See the BRC-118 guide for migration. + ### Removed (3.0.0 candidate) - Remove the legacy `identity/DIDTokenValidation` module and its diff --git a/packages/sdk/README.md b/packages/sdk/README.md index 27121f195..6b8e2072f 100644 --- a/packages/sdk/README.md +++ b/packages/sdk/README.md @@ -56,6 +56,25 @@ the most recent 1,000 entries. Simplified authenticated HTTP frames, bodies, headers, signatures, request IDs, and certificate-request headers have fixed size/count limits and redirects are rejected. +Ordinary HTTP fallback preserves native fetch body semantics: text, URLSearchParams, +typed Blob and FormData retain their generated media types and serialization. +Mutable inputs are snapshotted before discovery; authenticated payment retries +continue to use owned bytes. Explicit caller Content-Type headers take precedence. + +The next release adds negotiated BRC-118 multipart payments. AuthFetch prepares +with `noSend`, validates the actual final payment/request size, then submits that +same payment with `sendWith`. The wallet must support prepare/submit and +`abortAction`. Oversized header-only requests and multipart GET/HEAD requests +fail before broadcast; uncertain submission or delivery requires reconciliation +instead of another automatic spend. See the [BRC-118 guide](../../docs/guides/brc118-payments.md) +for receiver-first rollout, limits, cancellation, raw-byte payloads and typed +`PaymentTransportError` outcomes. Nonempty non-multipart authentication remains compatible; empty byte bodies use +the BRC-104 `-1` sentinel and require the matching auth middleware 2.3.0 receiver. + +BRC-29 receipt derives the recipient's own child key (`forSelf: true`). Independent +sender/recipient wallet tests protect this distinction; the payer's sibling +output is not a valid payment to the recipient. + SDK 2.8.10 accepts full WhatsOnChain block headers with up to 64 own data properties, matching the existing header-list response bound. Normal provider metadata no longer causes valid mainnet Merkle proofs to fail a 16-property @@ -98,8 +117,7 @@ output values remain unsigned and bounded. Applications affected by these client defects can update their bundled SDK without changing calls. Wallet upgrades continue to support the existing BRC100 contract; an ecosystem-wide application migration is not required. No API, wire -or account-data migration is required. Source 2.8.3 is not published until the -protected npm release workflow completes. +or account-data migration is required. These compatibility fixes are also included in the unpublished 3.0.0 candidate. ### Action history compatibility @@ -315,8 +333,8 @@ than that the script is invalid. `x-bsv-payment-known-txids` response header on its 402 challenge. The value is a comma-separated list of 64-character hexadecimal transaction IDs the recipient already possesses and has validated. `AuthFetch` passes at most - 256 unique lowercase IDs to the wallet's `createAction` options, including - when payment requirements change and a new transaction is created. This + 256 unique lowercase IDs to the wallet's prepared `createAction` options. + Changed payment requirements stop retries and require reconciliation. This lets compatible wallets omit known ancestors from payment BEEF. Whitespace, duplicates, and malformed entries are ignored; an absent or invalid-only header preserves existing payment behavior. Browser services must expose diff --git a/packages/sdk/src/auth/clients/AuthFetch.ts b/packages/sdk/src/auth/clients/AuthFetch.ts index 06813f6a5..8ac3871f3 100644 --- a/packages/sdk/src/auth/clients/AuthFetch.ts +++ b/packages/sdk/src/auth/clients/AuthFetch.ts @@ -5,7 +5,6 @@ import { toArray as UtilsToArray, toBase64, toHex as UtilsToHex, - toSafeString, toUTF8Strict } from '../../primitives/utils.js' import Random from '../../primitives/Random.js' @@ -29,6 +28,16 @@ import { VerifiableCertificate } from '../certificates/VerifiableCertificate.js' import { Writer } from '../../primitives/utils.js' import { getVerifiableCertificates } from '../utils/getVerifiableCertificates.js' import { copyAuthByteArray } from '../AuthMessageValidation.js' +import { + authenticatedContentType, + PaymentTransportError, + paymentTransports, + preparePaymentTransport, + resolvePaymentTransportLimits, + type PaymentTransportLimits +} from '../utils/paymentTransport.js' +import { Beef } from '../../transaction/Beef.js' +import { validateWalletResult } from '../../wallet/WalletResultValidation.js' interface SimplifiedFetchRequestOptions { method?: string @@ -37,6 +46,10 @@ interface SimplifiedFetchRequestOptions { retryCounter?: number paymentContext?: PaymentRetryContext paymentRetryAttempts?: number + /** BRC-118 wire limits. Infeasible payments are refused before broadcast. */ + paymentTransport?: PaymentTransportLimits + /** Cancellation before broadcast releases the prepared action when the wallet permits. */ + signal?: AbortSignal /** * Optional wallet action labels applied to BRC-105 payment transactions * created for 402 responses. Use these to find payments later via @@ -72,6 +85,14 @@ interface PaymentRetryContext { attempts: number maxAttempts: number errors: PaymentErrorLogEntry[] + txid?: string + state?: 'prepared' | 'submitted' | 'uncertain' + originalRequest?: { method: string; headers: Record; body?: Uint8Array } + preparedRequest?: { + headers: Record + body?: Uint8Array + transport: 'header' | 'multipart' + } requestSummary: { url: string method: string @@ -226,6 +247,19 @@ export class AuthFetch { * @throws Will throw an error if unsupported headers are used or other validation fails. */ async fetch(url: string, config: SimplifiedFetchRequestOptions = {}): Promise { + if (config.signal?.aborted === true) + throw new PaymentTransportError('ERR_PAYMENT_CANCELLED', 'Paid request cancelled.') + // Own mutable input before any network or wallet await. Keep native body + // types so the ordinary HTTP fallback retains fetch's media-type handling. + const headers = { ...config.headers } + const ownedBody = await this.snapshotRequestBody(config.body) + config = { + ...config, + headers, + body: ownedBody, + paymentTransport: { ...config.paymentTransport }, + labels: config.labels?.slice() + } if (typeof config.retryCounter === 'number') { if (config.retryCounter <= 0) { throw new Error('Request failed after maximum number of retries.') @@ -273,6 +307,7 @@ export class AuthFetch { } clearTimeout(responseTimeout) this.pendingRequestNonces.delete(requestNonceAsBase64) + config.signal?.removeEventListener('abort', cancelRequest) } const resolveRequest = (response: Response): void => { cleanup() @@ -282,6 +317,21 @@ export class AuthFetch { cleanup() reject(error) } + const cancelRequest = (): void => + rejectRequest( + new PaymentTransportError( + 'ERR_PAYMENT_CANCELLED', + 'Paid request cancelled.', + config.paymentContext == null + ? undefined + : { txid: config.paymentContext.txid, state: config.paymentContext.state } + ) + ) + config.signal?.addEventListener('abort', cancelRequest, { once: true }) + if (config.signal?.aborted === true) { + cancelRequest() + return + } this.pendingRequestNonces.add(requestNonceAsBase64) listenerId = peerToUse.peer.listenForGeneralMessages( @@ -311,7 +361,10 @@ export class AuthFetch { // A certificate prompt can outlive the request deadline. Never // dispatch a request after its caller has already seen a timeout. if (cleaned) return - await peerToUse.peer.toPeer(writer.toArray(), peerToUse.identityKey) + await peerToUse.peer.toPeer( + writer.toArray(), + config.paymentContext?.serverIdentityKey ?? peerToUse.identityKey + ) } catch (error) { // Late transport/session failures must not start recovery that // replays a request after its response deadline has expired. @@ -588,8 +641,8 @@ export class AuthFetch { if (normalizedKey.startsWith('x-bsv-auth')) { throw new Error('No BSV auth headers allowed here!') } - } else if (normalizedKey.startsWith('content-type')) { - value = value.split(';')[0].trim() + } else if (normalizedKey === 'content-type') { + value = authenticatedContentType(value) } else { throw new Error( 'Unsupported header in the simplified fetch implementation. Only content-type, authorization, and x-bsv-* headers are supported.' @@ -625,8 +678,8 @@ export class AuthFetch { return } const bytes = await this.normalizeBodyToNumberArray(body) - writer.writeVarIntNum(bytes.length) - writer.write(bytes) + writer.writeVarIntNum(bytes.length === 0 ? -1 : bytes.length) + if (bytes.length > 0) writer.write(bytes) } /** @@ -713,36 +766,8 @@ export class AuthFetch { config: SimplifiedFetchRequestOptions, originalResponse: Response ): Promise { - const paymentVersion = originalResponse.headers.get('x-bsv-payment-version') - if (!paymentVersion || paymentVersion !== PAYMENT_VERSION) { - throw new Error( - `Unsupported x-bsv-payment-version response header. Client version: ${PAYMENT_VERSION}, Server version: ${paymentVersion}` - ) - } - - const satoshisRequiredHeader = originalResponse.headers.get('x-bsv-payment-satoshis-required') - if (!satoshisRequiredHeader) { - throw new Error('Missing x-bsv-payment-satoshis-required response header.') - } - if (!/^[1-9]\d*$/.test(satoshisRequiredHeader)) { - throw new Error('Invalid x-bsv-payment-satoshis-required response header value.') - } - const satoshisRequired = Number(satoshisRequiredHeader) - if (!Number.isSafeInteger(satoshisRequired)) { - throw new Error('Invalid x-bsv-payment-satoshis-required response header value.') - } - - const serverIdentityKey = originalResponse.headers.get('x-bsv-auth-identity-key') - if (typeof serverIdentityKey !== 'string') { - throw new TypeError('Missing x-bsv-auth-identity-key response header.') - } - - const derivationPrefix = originalResponse.headers.get('x-bsv-payment-derivation-prefix') - if (typeof derivationPrefix !== 'string' || derivationPrefix.length < 1) { - throw new Error('Missing x-bsv-payment-derivation-prefix response header.') - } - - const knownTxids = parseKnownTxidsHeader(originalResponse.headers.get(KNOWN_TXIDS_HEADER)) + const { satoshisRequired, serverIdentityKey, derivationPrefix, knownTxids, transports } = + paymentRequirements(originalResponse) let paymentContext = config.paymentContext if (paymentContext == null) { @@ -752,7 +777,8 @@ export class AuthFetch { satoshisRequired, serverIdentityKey, derivationPrefix, - knownTxids + knownTxids, + transports ) } else { const requirementsChanged = !this.isPaymentContextCompatible( @@ -762,18 +788,10 @@ export class AuthFetch { derivationPrefix ) if (requirementsChanged) { - this.logPaymentAttempt( - 'warn', - 'Server adjusted payment requirements; regenerating transaction', - this.composePaymentLogDetails(url, paymentContext) - ) - paymentContext = await this.createPaymentContext( - url, - config, - satoshisRequired, - serverIdentityKey, - derivationPrefix, - knownTxids + throw new PaymentTransportError( + 'ERR_PAYMENT_REQUIREMENTS_CHANGED', + 'The server changed payment requirements after a payment was prepared. Reconcile the existing payment before authorizing another.', + { txid: paymentContext.txid, state: paymentContext.state } ) } } @@ -786,18 +804,32 @@ export class AuthFetch { ) } - const headersWithPayment: Record = { - ...config.headers + await this.prepareExistingPaymentRequest(paymentContext, config, transports) + + if (config.signal?.aborted === true) { + const aborted = + paymentContext.state === 'prepared' + ? await this.abortPreparedPayment(paymentContext.txid) + : undefined + throw new PaymentTransportError('ERR_PAYMENT_CANCELLED', 'Paid request cancelled.', { + txid: paymentContext.txid, + state: paymentContext.state, + aborted + }) + } + if (paymentContext.state === 'prepared') await this.submitPreparedPayment(paymentContext) + if (config.signal?.aborted === true) { + throw new PaymentTransportError( + 'ERR_PAYMENT_CANCELLED', + 'Paid request cancelled after submission; reconcile its outcome.', + { txid: paymentContext.txid, state: paymentContext.state } + ) } - headersWithPayment['x-bsv-payment'] = JSON.stringify({ - derivationPrefix: paymentContext.derivationPrefix, - derivationSuffix: paymentContext.derivationSuffix, - transaction: paymentContext.transactionBase64 - }) const nextConfig: SimplifiedFetchRequestOptions = { ...config, - headers: headersWithPayment, + headers: paymentContext.preparedRequest.headers, + body: paymentContext.preparedRequest.body, paymentContext } @@ -815,15 +847,53 @@ export class AuthFetch { attemptDetails ) + return this.sendPaidRequest( + url, + nextConfig, + originalResponse, + paymentContext, + attemptDetails, + attemptNumber + ) + } + + private async sendPaidRequest( + url: string, + nextConfig: SimplifiedFetchRequestOptions, + originalResponse: Response, + paymentContext: PaymentRetryContext, + attemptDetails: Record, + attemptNumber: number + ): Promise { try { const response = await this.fetch(url, nextConfig) + if (response.status === 413 || response.status === 431) { + throw new PaymentTransportError( + 'ERR_PAYMENT_SIZE', + 'The authenticated server rejected the paid request size.', + { txid: paymentContext.txid, state: paymentContext.state }, + response.status, + true + ) + } this.logPaymentAttempt( - 'info', - `Paid request attempt ${attemptNumber} succeeded`, + response.ok ? 'info' : 'warn', + `Paid request attempt ${attemptNumber} completed with HTTP ${response.status}`, attemptDetails ) return response } catch (error) { + if (error instanceof PaymentTransportError) throw error + const status = error instanceof Error ? (error as any).details?.status : undefined + if (status === 413 || status === 431) { + throw new PaymentTransportError( + 'ERR_PAYMENT_SIZE', + 'An unauthenticated intermediary rejected the paid request size. Reconcile the submitted payment.', + { txid: paymentContext.txid, state: paymentContext.state }, + status, + false + ) + } const errorEntry = this.createPaymentErrorEntry(paymentContext.attempts, error) paymentContext.errors.push(errorEntry) this.logPaymentAttempt('error', `Paid request attempt ${attemptNumber} failed`, { @@ -844,6 +914,53 @@ export class AuthFetch { } } + private async prepareExistingPaymentRequest( + paymentContext: PaymentRetryContext, + config: SimplifiedFetchRequestOptions, + transports: ReadonlySet + ): Promise { + // Contexts returned by the released API may already represent a spend. + // Preserve them without another wallet mutation, but validate their real wire bytes. + const legacy = paymentContext.preparedRequest === undefined + if (legacy) { + const beef = Beef.fromBinaryStrict(UtilsToArray(paymentContext.transactionBase64, 'base64')) + if (beef.atomicTxid == null) + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'Existing payment context requires Atomic BEEF.' + ) + paymentContext.txid = beef.atomicTxid + paymentContext.state = 'uncertain' + paymentContext.originalRequest = { + method: config.method ?? 'GET', + headers: { ...config.headers }, + body: + config.body == null + ? undefined + : new Uint8Array(await this.normalizeBodyToNumberArray(config.body)) + } + } else if (transports.has(paymentContext.preparedRequest.transport)) return + try { + paymentContext.preparedRequest = preparePaymentTransport( + JSON.stringify({ + derivationPrefix: paymentContext.derivationPrefix, + derivationSuffix: paymentContext.derivationSuffix, + transaction: paymentContext.transactionBase64 + }), + paymentContext.originalRequest, + transports, + resolvePaymentTransportLimits(config.paymentTransport) + ) + } catch (error) { + if (legacy) throw error + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'The authenticated server no longer supports a deliverable transport for this existing payment.', + { txid: paymentContext.txid, state: paymentContext.state } + ) + } + } + private isPaymentContextCompatible( context: PaymentRetryContext, satoshisRequired: number, @@ -863,11 +980,46 @@ export class AuthFetch { satoshisRequired: number, serverIdentityKey: string, derivationPrefix: string, - knownTxids?: string[] + knownTxids: string[] | undefined, + transports: ReadonlySet = new Set(['header']) ): Promise { const requestSummary = this.buildPaymentRequestSummary(url, config) const paymentLabels = Array.isArray(config.labels) ? [...config.labels] : [] const maxAttempts = this.getMaxPaymentAttempts(config) + const limits = resolvePaymentTransportLimits(config.paymentTransport) + if (typeof this.wallet.abortAction !== 'function') + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'Payment preparation requires a BRC-100 wallet with noSend, sendWith, and abortAction support.' + ) + if (Object.keys(config.headers ?? {}).some(name => name.toLowerCase() === 'x-bsv-payment')) { + throw new PaymentTransportError( + 'ERR_PAYMENT_REQUIREMENTS_CHANGED', + 'An existing payment was rejected. Reconcile it before authorizing another payment.' + ) + } + const originalRequest = { + method: config.method ?? 'GET', + headers: { ...config.headers }, + body: + config.body == null + ? undefined + : new Uint8Array(await this.normalizeBodyToNumberArray(config.body)) + } + if ( + !transports.has('header') && + ['GET', 'HEAD'].includes(originalRequest.method.toUpperCase()) + ) { + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'GET and HEAD require header payment support.' + ) + } + const { publicKey: clientIdentityKey } = validateWalletResult( + 'getPublicKey', + await this.wallet.getPublicKey({ identityKey: true }, this.originator), + { identityKey: true } + ) const derivationSuffix = await createNonce(this.wallet, undefined, this.originator) const { publicKey: derivedPublicKey } = await this.wallet.getPublicKey( @@ -882,58 +1034,145 @@ export class AuthFetch { .lock(PublicKey.fromString(derivedPublicKey).toAddress()) .toHex() - const { tx: transactionResult } = await this.wallet.createAction( - { - description: `Payment for request to ${new URL(url).origin}`, - labels: this.buildPaymentActionLabels( - { labels: paymentLabels }, - derivationPrefix, - derivationSuffix - ), - outputs: [ - { - satoshis: satoshisRequired, - lockingScript, - customInstructions: JSON.stringify({ - derivationPrefix, - derivationSuffix, - payee: serverIdentityKey - }), - outputDescription: 'HTTP request payment' - } - ], - options: { - randomizeOutputs: false, - // Ancestors the recipient already holds are emitted txid-only rather than in full. - // Undefined when the server did not declare any, which is the pre-existing behaviour. - ...(knownTxids != null ? { knownTxids } : {}) + const createArgs = { + description: 'BRC-105 HTTP request payment', + labels: this.buildPaymentActionLabels( + { labels: paymentLabels }, + derivationPrefix, + derivationSuffix + ), + outputs: [ + { + satoshis: satoshisRequired, + lockingScript, + customInstructions: JSON.stringify({ + derivationPrefix, + derivationSuffix, + payee: serverIdentityKey + }), + outputDescription: 'HTTP request payment' } - }, - this.originator - ) - - const transaction = copyAuthByteArray( - transactionResult, - 'BRC-105 payment transaction', - MAX_PAYMENT_TRANSACTION_BYTES + ], + options: { + randomizeOutputs: false, + noSend: true, + acceptDelayedBroadcast: false, + // Ancestors the recipient already holds are emitted txid-only rather than in full. + // Undefined when the server did not declare any, which is the pre-existing behaviour. + ...(knownTxids != null ? { knownTxids } : {}) + } + } + if (config.signal?.aborted === true) + throw new PaymentTransportError( + 'ERR_PAYMENT_CANCELLED', + 'Paid request cancelled before preparation.' + ) + const rawCreated = await this.wallet.createAction(createArgs, this.originator) + let txid: string | undefined + let reference: string | undefined + let preparationError: unknown + try { + // A malformed result can still identify a prepared reservation. Read only + // a canonical own data field, never an accessor supplied by an adapter. + const reported = + rawCreated == null ? undefined : Object.getOwnPropertyDescriptor(rawCreated, 'txid')?.value + if (typeof reported === 'string' && /^[0-9a-f]{64}$/.test(reported)) txid = reported + const created = validateWalletResult('createAction', rawCreated, createArgs) + reference = created.signableTransaction?.reference + + const transaction = copyAuthByteArray( + created.tx, + 'BRC-105 payment transaction', + MAX_PAYMENT_TRANSACTION_BYTES + ) + txid = preparedPaymentTxid(transaction, txid, satoshisRequired, lockingScript) + const transactionBase64 = toBase64(transaction) + const preparedRequest = preparePaymentTransport( + JSON.stringify({ derivationPrefix, derivationSuffix, transaction: transactionBase64 }), + originalRequest, + transports, + limits + ) + // Exercise the actual released wire decoder's caps before the irreversible sendWith call. + const frame = await this.serializeRequest( + originalRequest.method, + preparedRequest.headers, + preparedRequest.body, + new URL(url), + Random(32) + ) + this.#createTransport(new URL(url).origin).deserializeRequestPayload(frame.toArray()) + if (config.signal?.aborted === true) + throw new PaymentTransportError( + 'ERR_PAYMENT_CANCELLED', + 'Paid request cancelled during preparation.' + ) + return { + satoshisRequired, + transactionBase64, + derivationPrefix, + derivationSuffix, + serverIdentityKey, + clientIdentityKey, + attempts: 0, + maxAttempts, + errors: [], + requestSummary, + txid, + state: 'prepared', + originalRequest, + preparedRequest + } + } catch (error) { + preparationError = error + } + // A failed preparation may still own a reservation. Cleanup precedes the refusal. + const aborted = await this.abortPreparedPayment(txid ?? reference) + throw new PaymentTransportError( + preparationError instanceof PaymentTransportError + ? preparationError.code + : 'ERR_PAYMENT_TRANSPORT', + 'Payment preparation could not produce a deliverable request; no broadcast was requested.', + txid == null ? undefined : { txid, state: 'prepared', aborted } ) + } - const { publicKey: clientIdentityKey } = await this.wallet.getPublicKey( - { identityKey: true }, - this.originator - ) + private async abortPreparedPayment(reference: string | undefined): Promise { + if (reference == null) return false + try { + return (await this.wallet.abortAction({ reference }, this.originator)).aborted === true + } catch { + return false + } + } - return { - satoshisRequired, - transactionBase64: toBase64(transaction), - derivationPrefix, - derivationSuffix, - serverIdentityKey, - clientIdentityKey, - attempts: 0, - maxAttempts, - errors: [], - requestSummary + private async submitPreparedPayment(context: PaymentRetryContext): Promise { + // Once submission starts its result may be uncertain. Never abort or create a replacement automatically. + context.state = 'uncertain' + try { + const args = { + description: 'Submit prepared HTTP payment', + options: { sendWith: [context.txid], acceptDelayedBroadcast: false } + } + const result = validateWalletResult( + 'createAction', + await this.wallet.createAction(args, this.originator), + args + ) + if ( + result.sendWithResults?.length !== 1 || + result.sendWithResults[0].txid !== context.txid || + result.sendWithResults[0].status !== 'unproven' + ) { + throw new Error('Payment broadcast did not return affirmative acceptance.') + } + context.state = 'submitted' + } catch { + throw new PaymentTransportError( + 'ERR_PAYMENT_OUTCOME_UNKNOWN', + 'Payment submission was not confirmed. Reconcile this transaction before retrying.', + { txid: context.txid, state: context.state } + ) } } @@ -1114,21 +1353,14 @@ export class AuthFetch { } private createPaymentErrorEntry(attempt: number, error: unknown): PaymentErrorLogEntry { - const entry: PaymentErrorLogEntry = { + // Provider exceptions can echo URLs, request headers or payment bytes. Keep + // only a bounded category in telemetry and terminal diagnostic history. + return { attempt, timestamp: new Date().toISOString(), - message: '', + message: error instanceof Error ? 'Payment delivery failed.' : 'Payment delivery rejected.', stack: undefined } - - if (error instanceof Error) { - entry.message = error.message - entry.stack = error.stack ?? undefined - } else { - entry.message = toSafeString(error) - } - - return entry } private getPaymentRetryDelay(attempt: number): number { @@ -1151,7 +1383,11 @@ export class AuthFetch { ): Error { const safeUrl = this.#safeLogUrl(url) const message = `Paid request to ${safeUrl} failed after ${context.attempts}/${context.maxAttempts} attempts. Sent ${context.satoshisRequired} satoshis to ${context.serverIdentityKey}.` - const error = new Error(message) + const error = new PaymentTransportError( + 'ERR_PAYMENT_OUTCOME_UNKNOWN', + message, + context.txid == null ? undefined : { txid: context.txid, state: context.state ?? 'uncertain' } + ) const failureDetails = { request: context.requestSummary, @@ -1184,6 +1420,23 @@ export class AuthFetch { } } + private async snapshotRequestBody( + body: BodyInit | null | undefined + ): Promise { + if (body == null) return undefined + if (typeof body === 'string') return body + // Blobs (including Files) are immutable and retain their declared type. + if (typeof Blob !== 'undefined' && body instanceof Blob) return body + if (typeof URLSearchParams !== 'undefined' && body instanceof URLSearchParams) + return new URLSearchParams(body) + if (typeof FormData !== 'undefined' && body instanceof FormData) { + const snapshot = new FormData() + body.forEach((value, key) => snapshot.append(key, value)) + return snapshot + } + return new Uint8Array(await this.normalizeBodyToNumberArray(body)) + } + private async normalizeBodyToNumberArray(body: BodyInit | null | undefined): Promise { // 0. Null / undefined if (body == null) { @@ -1192,7 +1445,7 @@ export class AuthFetch { // 1. number[] if (Array.isArray(body) && body.every(item => typeof item === 'number')) { - return body // Return the array as is + return body.slice() } // 2. string @@ -1200,38 +1453,39 @@ export class AuthFetch { return UtilsToArray(body, 'utf8') } - // 3. ArrayBuffer / TypedArrays - if (body instanceof ArrayBuffer || ArrayBuffer.isView(body)) { - const typedArray = - body instanceof ArrayBuffer - ? new Uint8Array(body) - : new Uint8Array(body.buffer, body.byteOffset, body.byteLength) - return Array.from(typedArray) + // 3. ArrayBuffer / TypedArrays: preserve the exact view's byte offset and length. + if (body instanceof ArrayBuffer) body = new Uint8Array(body) + if (ArrayBuffer.isView(body)) { + return Array.from(new Uint8Array(body.buffer, body.byteOffset, body.byteLength)) } // 4. Blob - if (body instanceof Blob) { + if (typeof Blob !== 'undefined' && body instanceof Blob) { const arrayBuffer = await body.arrayBuffer() return Array.from(new Uint8Array(arrayBuffer)) } // 5. FormData - if (body instanceof FormData) { + if (typeof FormData !== 'undefined' && body instanceof FormData) { const entries: [string, string][] = [] body.forEach((value, key) => { - entries.push([key, typeof value === 'string' ? value : value.name]) + if (typeof value !== 'string') + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'Serialize file-bearing FormData to owned bytes with its Content-Type before authenticated fetch.' + ) + entries.push([key, value]) }) - const urlEncoded = new URLSearchParams(entries).toString() - return UtilsToArray(urlEncoded, 'utf8') + return UtilsToArray(new URLSearchParams(entries).toString(), 'utf8') } // 6. URLSearchParams - if (body instanceof URLSearchParams) { + if (typeof URLSearchParams !== 'undefined' && body instanceof URLSearchParams) { return UtilsToArray(body.toString(), 'utf8') } // 7. ReadableStream - if (body instanceof ReadableStream) { + if (typeof ReadableStream !== 'undefined' && body instanceof ReadableStream) { throw new TypeError('ReadableStream cannot be directly converted to number[].') } @@ -1243,6 +1497,70 @@ export class AuthFetch { } } +function paymentRequirements(originalResponse: Response) { + const paymentVersion = originalResponse.headers.get('x-bsv-payment-version') + if (!paymentVersion || paymentVersion !== PAYMENT_VERSION) { + throw new Error( + `Unsupported x-bsv-payment-version response header. Client version: ${PAYMENT_VERSION}, Server version: ${paymentVersion}` + ) + } + + const satoshisRequiredHeader = originalResponse.headers.get('x-bsv-payment-satoshis-required') + if (!satoshisRequiredHeader) { + throw new Error('Missing x-bsv-payment-satoshis-required response header.') + } + if (!/^[1-9]\d*$/.test(satoshisRequiredHeader)) { + throw new Error('Invalid x-bsv-payment-satoshis-required response header value.') + } + const satoshisRequired = Number(satoshisRequiredHeader) + if (!Number.isSafeInteger(satoshisRequired)) { + throw new Error('Invalid x-bsv-payment-satoshis-required response header value.') + } + + const serverIdentityKey = originalResponse.headers.get('x-bsv-auth-identity-key') + if (typeof serverIdentityKey !== 'string') { + throw new TypeError('Missing x-bsv-auth-identity-key response header.') + } + + const derivationPrefix = originalResponse.headers.get('x-bsv-payment-derivation-prefix') + if (typeof derivationPrefix !== 'string' || derivationPrefix.length < 1) { + throw new Error('Missing x-bsv-payment-derivation-prefix response header.') + } + + const knownTxids = parseKnownTxidsHeader(originalResponse.headers.get(KNOWN_TXIDS_HEADER)) + const transports = paymentTransports(originalResponse.headers.get('x-bsv-payment-transports')) + if (transports.size === 0) + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'The server advertised no supported payment transport.' + ) + + return { satoshisRequired, serverIdentityKey, derivationPrefix, knownTxids, transports } +} + +function preparedPaymentTxid( + transaction: number[], + reportedTxid: string | undefined, + satoshisRequired: number, + lockingScript: string +): string { + const beef = Beef.fromBinaryStrict(transaction) + const atomicTxid = beef.atomicTxid + const output = atomicTxid == null ? undefined : beef.findTxid(atomicTxid)?.tx?.outputs[0] + if ( + atomicTxid == null || + (reportedTxid != null && reportedTxid !== atomicTxid) || + output?.satoshis !== satoshisRequired || + output.lockingScript.toHex() !== lockingScript + ) { + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'The prepared payment does not match its authorized recipient and amount.' + ) + } + return atomicTxid +} + class StrictResponseReader { private readonly reader: Reader diff --git a/packages/sdk/src/auth/clients/__tests/paymentFixtures.ts b/packages/sdk/src/auth/clients/__tests/paymentFixtures.ts new file mode 100644 index 000000000..0e5b7941e --- /dev/null +++ b/packages/sdk/src/auth/clients/__tests/paymentFixtures.ts @@ -0,0 +1,34 @@ +import Transaction from '../../../transaction/Transaction.js' +import Script from '../../../script/Script.js' +import { toBase64 } from '../../../primitives/utils.js' +import type { CreateActionArgs, CreateActionResult } from '../../../wallet/Wallet.interfaces.js' + +/** Real Atomic BEEF keeps retry tests sensitive to the wire validation boundary. */ +export function paymentActionResult(args: CreateActionArgs): Promise { + // Execute immediately but turn construction errors into wallet-style Promise rejections. + return new Promise(resolve => { + if (args.options?.sendWith !== undefined) { + resolve({ + sendWithResults: args.options.sendWith.map(txid => ({ txid, status: 'unproven' as const })) + }) + return + } + const source = new Transaction() + source.addOutput({ satoshis: 1000, lockingScript: Script.fromASM('OP_TRUE') }) + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + tx.addOutput({ + satoshis: args.outputs![0].satoshis, + lockingScript: Script.fromHex(args.outputs![0].lockingScript) + }) + resolve({ txid: tx.id('hex'), tx: tx.toAtomicBEEF() }) + }) +} + +const legacy = new Transaction() +legacy.addOutput({ satoshis: 5, lockingScript: Script.fromASM('OP_TRUE') }) +export const legacyPaymentBase64 = toBase64(legacy.toAtomicBEEF()) diff --git a/packages/sdk/src/auth/clients/__tests__/AuthFetch.additional.test.ts b/packages/sdk/src/auth/clients/__tests__/AuthFetch.additional.test.ts index 34230b95a..98ece1835 100644 --- a/packages/sdk/src/auth/clients/__tests__/AuthFetch.additional.test.ts +++ b/packages/sdk/src/auth/clients/__tests__/AuthFetch.additional.test.ts @@ -365,11 +365,11 @@ describe('AuthFetch.handlePaymentAndRetry – header validation', () => { }) // --------------------------------------------------------------------------- -// 5. handlePaymentAndRetry – incompatible context triggers new context creation +// 5. handlePaymentAndRetry – incompatible context requires reconciliation // --------------------------------------------------------------------------- describe('AuthFetch.handlePaymentAndRetry – context compatibility', () => { - it('regenerates context when server changes payment requirements', async () => { + it('requires reconciliation when server changes payment requirements', async () => { const authFetch = new AuthFetch(buildWallet()) jest.spyOn(authFetch as any, 'logPaymentAttempt').mockImplementation(() => {}) jest.spyOn(authFetch as any, 'wait').mockResolvedValue(undefined) @@ -400,15 +400,15 @@ describe('AuthFetch.handlePaymentAndRetry – context compatibility', () => { .mockResolvedValue(new Response('ok', { status: 200 })) const response = make402Response({ 'x-bsv-payment-satoshis-required': '10' }) // changed from 5 - await (authFetch as any).handlePaymentAndRetry( - 'https://example.com', - { paymentContext: existingContext }, - response - ) - - // createNonce should have been called because the context was regenerated - expect(createNonceMock).toHaveBeenCalled() - expect(fetchSpy).toHaveBeenCalledTimes(1) + await expect( + (authFetch as any).handlePaymentAndRetry( + 'https://example.com', + { paymentContext: existingContext }, + response + ) + ).rejects.toMatchObject({ code: 'ERR_PAYMENT_REQUIREMENTS_CHANGED' }) + expect(createNonceMock).not.toHaveBeenCalled() + expect(fetchSpy).not.toHaveBeenCalled() }) }) @@ -742,11 +742,12 @@ describe('AuthFetch.normalizeBodyToNumberArray (private)', () => { expect(result).toEqual(Utils.toArray('name=alice', 'utf8')) }) - it('normalizes FormData file entries by filename', async () => { + it('rejects FormData files instead of silently replacing their bytes with filenames', async () => { const fd = new FormData() fd.append('upload', new Blob(['hello']), 'greeting.txt') - const result = await (authFetch as any).normalizeBodyToNumberArray(fd) - expect(result).toEqual(Utils.toArray('upload=greeting.txt', 'utf8')) + await expect((authFetch as any).normalizeBodyToNumberArray(fd)).rejects.toThrow( + 'Serialize file-bearing FormData' + ) }) it('normalizes URLSearchParams bytes', async () => { @@ -1128,24 +1129,26 @@ describe('AuthFetch.createPaymentErrorEntry (private)', () => { authFetch = new AuthFetch(buildWallet()) }) - it('extracts message and stack from an Error instance', () => { - const err = new Error('something went wrong') + it('redacts provider messages and stack traces', () => { + const err = new Error('https://user:secret@example.com/private?beef=private-transaction') const entry = (authFetch as any).createPaymentErrorEntry(2, err) expect(entry.attempt).toBe(2) - expect(entry.message).toBe('something went wrong') - expect(typeof entry.stack).toBe('string') + expect(entry.message).toBe('Payment delivery failed.') + expect(JSON.stringify(entry)).not.toContain('secret') + expect(JSON.stringify(entry)).not.toContain('private-transaction') + expect(entry.stack).toBeUndefined() expect(typeof entry.timestamp).toBe('string') }) - it('converts non-Error to string message', () => { + it('redacts non-Error rejection values', () => { const entry = (authFetch as any).createPaymentErrorEntry(1, 'just a string error') - expect(entry.message).toBe('just a string error') + expect(entry.message).toBe('Payment delivery rejected.') expect(entry.stack).toBeUndefined() }) - it('converts numeric error to string message', () => { + it('categorizes numeric rejection values', () => { const entry = (authFetch as any).createPaymentErrorEntry(1, 42) - expect(entry.message).toBe('42') + expect(entry.message).toBe('Payment delivery rejected.') }) }) diff --git a/packages/sdk/src/auth/clients/__tests__/AuthFetch.boundary.test.ts b/packages/sdk/src/auth/clients/__tests__/AuthFetch.boundary.test.ts index 5afba1e3b..7342ec262 100644 --- a/packages/sdk/src/auth/clients/__tests__/AuthFetch.boundary.test.ts +++ b/packages/sdk/src/auth/clients/__tests__/AuthFetch.boundary.test.ts @@ -284,6 +284,77 @@ describe('AuthFetch pending-request boundary', () => { expect((authFetch as any).pendingRequestNonces.size).toBe(0) }) + test.each([ + ['serialization', false], + ['serialization', true], + ['certificate wait', false], + ['certificate wait', true] + ] as const)( + 'cancellation during %s drains state and preserves existing payment context (%s)', + async (phase, paid) => { + const controller = new AbortController() + const remove = jest.spyOn(controller.signal, 'removeEventListener') + const peer = { + listenForGeneralMessages: jest.fn(() => 52), + stopListeningForGeneralMessages: jest.fn(), + toPeer: jest.fn() + } + const authFetch = new AuthFetch({} as never) + ;(authFetch as any).peers['https://service.example'] = { + peer, + identityKey: 'server-identity-key', + supportsMutualAuth: true, + pendingCertificateRequests: phase === 'certificate wait' ? [true] : [] + } + let entered!: () => void + let release!: () => void + const started = new Promise(resolve => { + entered = resolve + }) + const gate = new Promise(resolve => { + release = resolve + }) + if (phase === 'serialization') { + const serialize = (authFetch as any).serializeRequest.bind(authFetch) + jest + .spyOn(authFetch as any, 'serializeRequest') + .mockImplementation(async (...args: unknown[]) => { + const result = await serialize(...args) + entered() + await gate + return result + }) + } else { + jest + .spyOn(authFetch as any, 'waitForPendingCertificateRequests') + .mockImplementation(async () => { + entered() + await gate + }) + } + const payment = paid ? { txid: 'ab'.repeat(32), state: 'submitted' } : undefined + const request = authFetch.fetch('https://service.example/resource', { + signal: controller.signal, + paymentContext: payment as any + }) + const rejected = expect(request).rejects.toMatchObject({ + code: 'ERR_PAYMENT_CANCELLED', + message: 'Paid request cancelled.', + payment + }) + await started + controller.abort() + release() + await rejected + expect(peer.toPeer).not.toHaveBeenCalled() + expect(peer.stopListeningForGeneralMessages).toHaveBeenCalledTimes( + phase === 'serialization' ? 0 : 1 + ) + expect(remove).toHaveBeenCalledWith('abort', expect.any(Function)) + expect((authFetch as any).pendingRequestNonces.size).toBe(0) + } + ) + test('times out and cleans an authenticated request with no response', async () => { jest.useFakeTimers() try { @@ -502,7 +573,8 @@ describe('AuthFetch pending-request boundary', () => { recursiveResponse ) expect(fetchSpy).toHaveBeenCalledTimes(2) - expect(config.retryCounter).toBe(3) + expect(fetchSpy.mock.calls[1][1]?.retryCounter).toBe(3) + expect(config.retryCounter).toBeUndefined() expect(authFetch.peers['https://service.example']).toBeUndefined() expect(stopListeningForGeneralMessages).toHaveBeenCalledWith(46) expect((authFetch as any).pendingRequestNonces.size).toBe(0) diff --git a/packages/sdk/src/auth/clients/__tests__/AuthFetch.knownTxids.test.ts b/packages/sdk/src/auth/clients/__tests__/AuthFetch.knownTxids.test.ts index 733c0628c..1e4c54ac6 100644 --- a/packages/sdk/src/auth/clients/__tests__/AuthFetch.knownTxids.test.ts +++ b/packages/sdk/src/auth/clients/__tests__/AuthFetch.knownTxids.test.ts @@ -3,6 +3,7 @@ import { parseKnownTxidsHeader, AuthFetch } from '../AuthFetch.js' import { Utils, PrivateKey } from '../../../primitives/index.js' import type { CreateActionOptions, WalletInterface } from '../../../wallet/Wallet.interfaces.js' import type { Peer } from '../../Peer.js' +import { paymentActionResult } from '../__tests/paymentFixtures.js' jest.mock('../../utils/createNonce.js', () => ({ createNonce: jest.fn() @@ -13,7 +14,9 @@ import { createNonce } from '../../utils/createNonce.js' const createNonceMock = createNonce as jest.MockedFunction type FetchOptions = NonNullable[1]> type PaymentContext = NonNullable -type TestWallet = jest.Mocked> +type TestWallet = jest.Mocked< + Pick +> interface PaymentInternals { handlePaymentAndRetry: ( @@ -99,9 +102,8 @@ function buildWallet(): TestWallet { .mockImplementation(async opts => opts?.identityKey === true ? { publicKey: identityKey } : { publicKey: derivedKey } ), - createAction: jest.fn().mockResolvedValue({ - tx: Utils.toArray('mock-tx', 'utf8') - }), + createAction: jest.fn(paymentActionResult), + abortAction: jest.fn().mockResolvedValue({ aborted: true }), createHmac: jest.fn().mockResolvedValue({ hmac: Array.from({ length: 32 }, () => 0) }) @@ -156,7 +158,9 @@ describe('AuthFetch.handlePaymentAndRetry – known-txids wiring', () => { } function optionsOfLastCreateAction(wallet: TestWallet): CreateActionOptions { - const options = wallet.createAction.mock.calls.at(-1)?.[0].options + const options = wallet.createAction.mock.calls.find( + ([args]) => args.options?.noSend === true + )?.[0].options if (options === undefined) throw new Error('Expected createAction options') return options } @@ -198,8 +202,12 @@ describe('AuthFetch.handlePaymentAndRetry – known-txids wiring', () => { ) expect(response?.status).toBe(200) - expect(wallet.createAction).toHaveBeenCalledTimes(1) - expect(optionsOfLastCreateAction(wallet)).toEqual({ randomizeOutputs: false }) + expect(wallet.createAction).toHaveBeenCalledTimes(2) + expect(optionsOfLastCreateAction(wallet)).toEqual({ + randomizeOutputs: false, + noSend: true, + acceptDelayedBroadcast: false + }) }) it.each([ @@ -227,7 +235,7 @@ describe('AuthFetch.handlePaymentAndRetry – known-txids wiring', () => { ) expect(response?.status).toBe(200) - expect(wallet.createAction).toHaveBeenCalledTimes(1) + expect(wallet.createAction).toHaveBeenCalledTimes(2) expect(optionsOfLastCreateAction(wallet).knownTxids).toEqual([A]) expect(createNonceMock).toHaveBeenCalledTimes(1) expect(fetchSpy).toHaveBeenCalledTimes(2) @@ -287,9 +295,11 @@ describe('AuthFetch.handlePaymentAndRetry – known-txids wiring', () => { const response = await authFetch.fetch('https://example.com/resource') expect(response.status).toBe(200) - expect(wallet.createAction).toHaveBeenCalledTimes(1) + expect(wallet.createAction).toHaveBeenCalledTimes(2) expect(optionsOfLastCreateAction(wallet)).toEqual({ randomizeOutputs: false, + noSend: true, + acceptDelayedBroadcast: false, knownTxids: [A, B] }) expect(wallet.getPublicKey).toHaveBeenCalledWith( @@ -300,22 +310,15 @@ describe('AuthFetch.handlePaymentAndRetry – known-txids wiring', () => { expect(peer.stopListeningForGeneralMessages).toHaveBeenCalledTimes(2) }) - it('forwards the declared txids when the server changes its price mid-flight', async () => { - // The regeneration branch builds a SECOND transaction. It is the path that matters most: - // a repriced retry is already the largest request in the exchange, so dropping the - // optimisation here would re-ship full ancestry at exactly the wrong moment. + it('requires reconciliation when the server changes price instead of creating another payment', async () => { const { internals, wallet } = harness() - - await internals.handlePaymentAndRetry( - 'https://example.com', - { paymentContext: existingContext(5) }, // server now asks for 10 - make402Response({ - 'x-bsv-payment-satoshis-required': '10', - 'x-bsv-payment-known-txids': A - }) - ) - - expect(wallet.createAction).toHaveBeenCalledTimes(1) - expect(optionsOfLastCreateAction(wallet).knownTxids).toEqual([A]) + await expect( + internals.handlePaymentAndRetry( + 'https://example.com', + { paymentContext: existingContext(5) }, + make402Response({ 'x-bsv-payment-satoshis-required': '10', 'x-bsv-payment-known-txids': A }) + ) + ).rejects.toMatchObject({ code: 'ERR_PAYMENT_REQUIREMENTS_CHANGED' }) + expect(wallet.createAction).not.toHaveBeenCalled() }) }) diff --git a/packages/sdk/src/auth/clients/__tests__/AuthFetch.multipart.test.ts b/packages/sdk/src/auth/clients/__tests__/AuthFetch.multipart.test.ts new file mode 100644 index 000000000..f383e8401 --- /dev/null +++ b/packages/sdk/src/auth/clients/__tests__/AuthFetch.multipart.test.ts @@ -0,0 +1,253 @@ +import { AuthFetch } from '../AuthFetch.js' +import PrivateKey from '../../../primitives/PrivateKey.js' +import Transaction from '../../../transaction/Transaction.js' +import Script from '../../../script/Script.js' +import { ProtoWallet } from '../../../wallet/ProtoWallet.js' +import { toBase64 } from '../../../primitives/utils.js' +import type { WalletInterface, CreateActionArgs } from '../../../wallet/Wallet.interfaces.js' + +const key = new PrivateKey(17) +const identity = key.toPublicKey().toString() +const prefix = toBase64(Array.from({ length: 48 }, () => 1)) +function setup(ancestorBytes = 0) { + const wallet = new ProtoWallet(key) as unknown as WalletInterface + const prepared: string[] = [] + wallet.abortAction = jest.fn(async () => ({ aborted: true })) + wallet.createAction = jest.fn(async (args: CreateActionArgs) => { + if (args.options?.sendWith != null) + return { + sendWithResults: args.options.sendWith.map(txid => ({ txid, status: 'unproven' as const })) + } + const source = new Transaction() + source.addOutput({ satoshis: 1000, lockingScript: Script.fromASM('OP_TRUE') }) + if (ancestorBytes > 0) + source.addOutput({ + satoshis: 0, + lockingScript: Script.fromASM(`OP_FALSE OP_RETURN ${'01'.repeat(ancestorBytes)}`) + }) + const payment = new Transaction() + payment.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + payment.addOutput({ + satoshis: args.outputs![0].satoshis, + lockingScript: Script.fromHex(args.outputs![0].lockingScript) + }) + prepared.push(payment.id('hex')) + return { txid: payment.id('hex'), tx: payment.toAtomicBEEF() } + }) + const client = new AuthFetch(wallet) + const internal = client as any + jest.spyOn(internal, 'logPaymentAttempt').mockImplementation(() => {}) + jest.spyOn(internal, 'wait').mockResolvedValue(undefined) + const send = jest.spyOn(client, 'fetch').mockResolvedValue(new Response('ok')) + const response = (transports?: string) => + new Response('', { + status: 402, + headers: { + 'x-bsv-payment-version': '1.0', + 'x-bsv-payment-satoshis-required': '10', + 'x-bsv-payment-derivation-prefix': prefix, + 'x-bsv-auth-identity-key': identity, + ...(transports === undefined ? {} : { 'x-bsv-payment-transports': transports }) + } + }) + const pay = (config = {}, advertisement?: string) => + internal.handlePaymentAndRetry( + 'https://payment.example/paid?q=unchanged', + config, + response(advertisement) + ) + return { client, internal, wallet, prepared, send, response, pay } +} + +afterEach(() => jest.restoreAllMocks()) + +describe('AuthFetch prepared BRC-118 payments', () => { + it('prepares without broadcast, keeps small payments in headers, then submits exactly once', async () => { + const { pay, wallet, send, prepared } = setup() + await pay() + expect(wallet.createAction).toHaveBeenCalledTimes(2) + expect(wallet.createAction).toHaveBeenNthCalledWith( + 1, + expect.objectContaining({ + options: expect.objectContaining({ noSend: true, randomizeOutputs: false }) + }), + undefined + ) + expect(wallet.createAction).toHaveBeenNthCalledWith( + 2, + expect.objectContaining({ + options: { sendWith: [prepared[0]], acceptDelayedBroadcast: false } + }), + undefined + ) + expect(send.mock.calls[0][1]!.headers!['x-bsv-payment']).toContain('transaction') + }) + it('wraps growing BEEF with the original bytes and media type', async () => { + const { pay, send, wallet } = setup(12_000) + await pay( + { + method: 'POST', + headers: { 'Content-Type': 'application/json; charset=utf-8' }, + body: '{ "utf8": "雪" }\n' + }, + 'header,multipart' + ) + const config = send.mock.calls[0][1]! + expect(config.method).toBe('POST') + expect(config.headers!['content-type']).toMatch(/^multipart\/form-data; boundary=/) + expect(config.headers!['x-bsv-payment']).toBeUndefined() + expect(Buffer.from(config.body).toString()).toContain( + 'Content-Type: application/json; charset=utf-8\r\n\r\n{ "utf8": "雪" }\n' + ) + expect(wallet.abortAction).not.toHaveBeenCalled() + }) + it.each([ + [{ method: 'POST' }, undefined], + [{ method: 'GET' }, 'header,multipart'], + [{ method: 'HEAD' }, 'header,multipart'], + [{ method: 'POST', paymentTransport: { maxBodyBytes: 1000 } }, 'header,multipart'], + [{ method: 'POST', paymentTransport: { maxPaymentBytes: 1000 } }, 'header,multipart'] + ])( + 'refuses an infeasible prepared payment before broadcasting %#', + async (config, advertisement) => { + const { pay, wallet, send, prepared } = setup(12_000) + await expect(pay(config, advertisement)).rejects.toMatchObject({ + retryable: false, + payment: { txid: expect.any(String), state: 'prepared', aborted: true } + }) + expect(wallet.createAction).toHaveBeenCalledTimes(1) + expect(wallet.abortAction).toHaveBeenCalledWith({ reference: prepared[0] }, undefined) + expect(send).not.toHaveBeenCalled() + } + ) + it('refuses unknown transport advertisements before touching the wallet', async () => { + const { pay, wallet } = setup() + await expect(pay({}, 'unknown')).rejects.toMatchObject({ code: 'ERR_PAYMENT_TRANSPORT' }) + expect(wallet.createAction).not.toHaveBeenCalled() + }) + it('reuses one payment and one frozen multipart body after a lost response', async () => { + const { pay, send, wallet } = setup(12_000) + send + .mockRejectedValueOnce(new Error('connection lost')) + .mockResolvedValueOnce(new Response('ok')) + await pay({ method: 'POST' }, 'header,multipart') + expect(wallet.createAction).toHaveBeenCalledTimes(2) + expect(send).toHaveBeenCalledTimes(2) + expect(send.mock.calls[0][1]!.body).toBe(send.mock.calls[1][1]!.body) + expect(send.mock.calls[0][1]!.headers).toBe(send.mock.calls[1][1]!.headers) + }) + it('never creates another spend when a paid retry receives a different challenge', async () => { + const { pay, send, internal, response, wallet } = setup() + send.mockImplementationOnce(async (url, config) => { + const changed = response('header,multipart') + changed.headers.set('x-bsv-payment-satoshis-required', '11') + return internal.handlePaymentAndRetry(url, config, changed) + }) + await expect(pay()).rejects.toMatchObject({ + code: 'ERR_PAYMENT_REQUIREMENTS_CHANGED', + payment: { state: 'submitted' } + }) + expect(wallet.createAction).toHaveBeenCalledTimes(2) + expect(wallet.abortAction).not.toHaveBeenCalled() + }) + it.each([true, false])( + 'treats size refusal as terminal, authenticated=%s', + async authenticated => { + const { pay, send, wallet } = setup() + if (authenticated) send.mockResolvedValueOnce(new Response('', { status: 413 })) + else + send.mockRejectedValueOnce( + Object.assign(new Error('proxy refused'), { details: { status: 431 } }) + ) + await expect(pay()).rejects.toMatchObject({ + code: 'ERR_PAYMENT_SIZE', + authenticated, + payment: { state: 'submitted' } + }) + expect(send).toHaveBeenCalledTimes(1) + expect(wallet.createAction).toHaveBeenCalledTimes(2) + expect(wallet.abortAction).not.toHaveBeenCalled() + } + ) + it('preserves an uncertain broadcast outcome without sending or automatically aborting', async () => { + const { pay, wallet, send } = setup() + const original = wallet.createAction + wallet.createAction = jest.fn(async args => { + if (args.options?.sendWith !== undefined) throw new Error('ack lost') + return original(args) + }) + await expect(pay()).rejects.toMatchObject({ + code: 'ERR_PAYMENT_OUTCOME_UNKNOWN', + payment: { state: 'uncertain' } + }) + expect(send).not.toHaveBeenCalled() + expect(wallet.abortAction).not.toHaveBeenCalled() + }) + it.each(['prepare', 'submit', 'retry'])( + 'cancels during %s without creating an extra spend', + async phase => { + const { pay, wallet, send } = setup() + const controller = new AbortController() + const original = wallet.createAction + wallet.createAction = jest.fn(async args => { + const result = await original(args) + if ( + (phase === 'prepare' && args.options?.noSend) || + (phase === 'submit' && args.options?.sendWith) + ) + controller.abort() + return result + }) + if (phase === 'retry') + send.mockImplementationOnce(async () => { + controller.abort() + throw new Error('lost response') + }) + await expect(pay({ signal: controller.signal })).rejects.toMatchObject({ + code: 'ERR_PAYMENT_CANCELLED', + payment: { state: phase === 'prepare' ? 'prepared' : 'submitted' } + }) + expect(wallet.createAction).toHaveBeenCalledTimes(phase === 'prepare' ? 1 : 2) + expect(wallet.abortAction).toHaveBeenCalledTimes(phase === 'prepare' ? 1 : 0) + expect(send).toHaveBeenCalledTimes(phase === 'retry' ? 1 : 0) + } + ) + it('changes an advertised transport without creating or submitting another payment', async () => { + const { pay, send, internal, response, wallet } = setup() + send.mockImplementationOnce(async (url, config) => + internal.handlePaymentAndRetry(url, config, response('multipart')) + ) + await pay({ method: 'POST', body: new Uint8Array([0, 1, 2]) }, 'header,multipart') + expect(send.mock.calls[0][1]!.headers!['x-bsv-payment']).toBeDefined() + expect(send.mock.calls[1][1]!.headers!['content-type']).toMatch(/^multipart/) + expect(wallet.createAction).toHaveBeenCalledTimes(2) + }) + it('retains payment context when multipart capability disappears after submission', async () => { + const { pay, send, internal, response, wallet } = setup(12_000) + send.mockImplementationOnce(async (url, config) => + internal.handlePaymentAndRetry(url, config, response('header')) + ) + await expect(pay({ method: 'POST' }, 'header,multipart')).rejects.toMatchObject({ + code: 'ERR_PAYMENT_TRANSPORT', + payment: { state: 'submitted' } + }) + expect(wallet.createAction).toHaveBeenCalledTimes(2) + expect(wallet.abortAction).not.toHaveBeenCalled() + }) + it('releases an identifiable reservation after a malformed wallet result', async () => { + const { pay, wallet, send } = setup() + const original = wallet.createAction + wallet.createAction = jest.fn(async args => ({ ...(await original(args)), tx: [-1] })) + await expect(pay()).rejects.toMatchObject({ + code: 'ERR_PAYMENT_TRANSPORT', + payment: { state: 'prepared', aborted: true } + }) + expect(wallet.createAction).toHaveBeenCalledTimes(1) + expect(wallet.abortAction).toHaveBeenCalledTimes(1) + expect(send).not.toHaveBeenCalled() + }) +}) diff --git a/packages/sdk/src/auth/clients/__tests__/AuthFetch.nativeBody.test.ts b/packages/sdk/src/auth/clients/__tests__/AuthFetch.nativeBody.test.ts new file mode 100644 index 000000000..a247b5d37 --- /dev/null +++ b/packages/sdk/src/auth/clients/__tests__/AuthFetch.nativeBody.test.ts @@ -0,0 +1,102 @@ +import { jest } from '@jest/globals' +import { AuthFetch } from '../AuthFetch.js' + +const url = 'https://service.example/form' + +function setup(cached: boolean) { + let request: Request + const fetchClient = jest.fn(async (input, init) => { + request = new Request(input, init) + return new Response('ok') + }) + const client = new AuthFetch({} as never, undefined, undefined, undefined, {}, fetchClient) + const peer = { + listenForGeneralMessages: jest.fn(() => 1), + stopListeningForGeneralMessages: jest.fn(), + toPeer: jest.fn(async () => { + throw new Error('HTTP server failed to authenticate') + }) + } + client.peers[new URL(url).origin] = { + peer: peer as never, + supportsMutualAuth: cached ? false : undefined, + pendingCertificateRequests: [] + } + return { client, fetchClient, request: () => request!, peer } +} + +describe.each([true, false])('ordinary HTTP native bodies (cached=%s)', cached => { + test.each([ + [ + 'URLSearchParams', + () => new URLSearchParams([['name', '雪 & space']]), + 'application/x-www-form-urlencoded;charset=UTF-8', + 'name=%E9%9B%AA+%26+space' + ], + [ + 'typed Blob', + () => new Blob(['{"name":"雪"}'], { type: 'application/json' }), + 'application/json', + '{"name":"雪"}' + ], + ['text', () => '雪 & space', 'text/plain;charset=UTF-8', '雪 & space'] + ])( + 'retains native %s serialization and inferred content type', + async (_name, body, type, text) => { + const fixture = setup(cached) + await fixture.client.fetch(url, { method: 'POST', body: body() }) + expect(fixture.request().headers.get('content-type')).toBe(type) + await expect(fixture.request().text()).resolves.toBe(text) + expect(fixture.request().redirect).toBe('error') + expect(fixture.peer.toPeer).toHaveBeenCalledTimes(cached ? 0 : 1) + expect(fixture.client.peers[new URL(url).origin].supportsMutualAuth).toBe(false) + } + ) + + test('retains multipart fields and snapshots FormData before awaiting discovery', async () => { + const fixture = setup(cached) + const body = new FormData() + body.append('name', '雪 & space') + body.append('name', 'second') + const pending = fixture.client.fetch(url, { method: 'POST', body }) + body.set('name', 'changed') + await pending + expect(fixture.request().headers.get('content-type')).toMatch( + /^multipart\/form-data; boundary=/ + ) + const received = await fixture.request().formData() + expect(received.getAll('name')).toEqual(['雪 & space', 'second']) + }) + + test('snapshots URLSearchParams while preserving explicit content type', async () => { + const fixture = setup(cached) + const body = new URLSearchParams({ name: 'original' }) + const headers = { 'Content-Type': 'application/custom' } + const pending = fixture.client.fetch(url, { method: 'POST', headers, body }) + body.set('name', 'changed') + headers['Content-Type'] = 'changed' + await pending + expect(fixture.request().headers.get('content-type')).toBe('application/custom') + await expect(fixture.request().text()).resolves.toBe('name=original') + }) + + test('retains owned bytes for a mutable array-buffer view', async () => { + const fixture = setup(cached) + const buffer = Uint8Array.of(0, 1, 2, 3) + const body = buffer.subarray(1, 3) + const pending = fixture.client.fetch(url, { method: 'POST', body }) + buffer.fill(9) + await pending + expect(fixture.request().headers.has('content-type')).toBe(false) + await expect(fixture.request().arrayBuffer()).resolves.toEqual(Uint8Array.of(1, 2).buffer) + }) + + test('snapshots numeric byte arrays before its first await', async () => { + const fixture = setup(cached) + const body = [1, 2] + const pending = fixture.client.fetch(url, { method: 'POST', body }) + body.fill(9) + await pending + await expect(fixture.request().arrayBuffer()).resolves.toEqual(Uint8Array.of(1, 2).buffer) + }) +}) diff --git a/packages/sdk/src/auth/clients/__tests__/AuthFetch.test.ts b/packages/sdk/src/auth/clients/__tests__/AuthFetch.test.ts index c3c62a4ab..5441d62ae 100644 --- a/packages/sdk/src/auth/clients/__tests__/AuthFetch.test.ts +++ b/packages/sdk/src/auth/clients/__tests__/AuthFetch.test.ts @@ -1,6 +1,8 @@ import { jest } from '@jest/globals' import { AuthFetch } from '../AuthFetch.js' import { Utils, PrivateKey } from '../../../primitives/index.js' +import { paymentActionResult, legacyPaymentBase64 } from '../__tests/paymentFixtures.js' +import { Beef } from '../../../transaction/Beef.js' jest.mock('../../utils/createNonce.js', () => ({ createNonce: jest.fn() @@ -49,9 +51,8 @@ function createWalletStub(): any { } return { publicKey: derivedKey } }), - createAction: jest.fn(async () => ({ - tx: Utils.toArray('mock-transaction', 'utf8') - })), + createAction: jest.fn(paymentActionResult), + abortAction: jest.fn(async () => ({ aborted: true })), createHmac: jest.fn(async () => ({ hmac: Array.from({ length: 32 }).fill(7) })) @@ -99,9 +100,9 @@ describe('AuthFetch payment handling', () => { expect(context.serverIdentityKey).toBe('remote-identity-key') expect(context.derivationPrefix).toBe('test-prefix') expect(context.derivationSuffix).toBe('suffix-from-test') - expect(context.transactionBase64).toBe( - Utils.toBase64(Utils.toArray('mock-transaction', 'utf8')) - ) + expect( + Beef.fromBinaryStrict(Utils.toArray(context.transactionBase64, 'base64')).atomicTxid + ).toEqual(expect.any(String)) expect(context.clientIdentityKey).toEqual(expect.any(String)) expect(context.attempts).toBe(0) expect(context.maxAttempts).toBe(3) @@ -121,7 +122,7 @@ describe('AuthFetch payment handling', () => { expect(wallet.createAction).toHaveBeenCalledWith( expect.objectContaining({ - description: expect.stringContaining('https://api.example.com'), + description: 'BRC-105 HTTP request payment', labels: [`brc105 ${prefixHex} ${suffixHex}`], outputs: [ expect.objectContaining({ @@ -177,7 +178,7 @@ describe('AuthFetch payment handling', () => { 'remote-identity-key', 'test-prefix' ) - ).rejects.toThrow('dense byte array') + ).rejects.toMatchObject({ code: 'ERR_PAYMENT_TRANSPORT' }) }) test('brc105 payment label hex survives lowercasing and round-trips to base64', () => { @@ -201,7 +202,7 @@ describe('AuthFetch payment handling', () => { const paymentContext: TestPaymentContext = { satoshisRequired: 5, - transactionBase64: Utils.toBase64([1, 2, 3]), + transactionBase64: legacyPaymentBase64, derivationPrefix: 'prefix', derivationSuffix: 'suffix', serverIdentityKey: 'server-key', @@ -248,7 +249,7 @@ describe('AuthFetch payment handling', () => { expect(paymentHeader).toEqual({ derivationPrefix: 'prefix', derivationSuffix: 'suffix', - transaction: Utils.toBase64([1, 2, 3]) + transaction: legacyPaymentBase64 }) expect(createPaymentContextSpy).not.toHaveBeenCalled() @@ -269,7 +270,7 @@ describe('AuthFetch payment handling', () => { const paymentContext: TestPaymentContext = { satoshisRequired: 5, - transactionBase64: Utils.toBase64([9, 9, 9]), + transactionBase64: legacyPaymentBase64, derivationPrefix: 'prefix', derivationSuffix: 'suffix', serverIdentityKey: 'server-key', @@ -314,13 +315,13 @@ describe('AuthFetch payment handling', () => { expect(err.details.errors[0]).toEqual( expect.objectContaining({ attempt: 1, - message: 'payment attempt 1 failed' + message: 'Payment delivery failed.' }) ) expect(err.details.errors[1]).toEqual( expect.objectContaining({ attempt: 2, - message: 'payment attempt 2 failed' + message: 'Payment delivery failed.' }) ) expect(typeof err.details.errors[0].timestamp).toBe('string') diff --git a/packages/sdk/src/auth/utils/__tests/decodePaymentPayload.test.ts b/packages/sdk/src/auth/utils/__tests/decodePaymentPayload.test.ts new file mode 100644 index 000000000..e8c39276b --- /dev/null +++ b/packages/sdk/src/auth/utils/__tests/decodePaymentPayload.test.ts @@ -0,0 +1,29 @@ +import { decodePaymentPayload } from '../decodePaymentPayload' + +const bytes = (text: string) => new TextEncoder().encode(text) +describe('authenticated payload decoding', () => { + it('distinguishes absent, empty, text, JSON and opaque nested bodies', () => { + expect(decodePaymentPayload(undefined, 'application/json')).toBeUndefined() + expect(decodePaymentPayload(bytes(''), 'text/plain')).toBe('') + expect(decodePaymentPayload(bytes('雪'), 'text/plain; charset=utf-8')).toBe('雪') + expect(decodePaymentPayload(bytes('{ "a": 1 }\n'), 'application/problem+json')).toEqual({ + a: 1 + }) + const binary = new Uint8Array([0, 255, 128]) + expect(decodePaymentPayload(binary, 'multipart/form-data; boundary=inner')).toBe(binary) + expect(decodePaymentPayload(binary, undefined)).toBe(binary) + expect(() => decodePaymentPayload(binary, 'text/plain')).toThrow() + expect(() => decodePaymentPayload(bytes('{'), 'application/json')).toThrow() + }) + it('retains duplicate form fields and prototype-shaped names without quadratic copying', () => { + const count = 20_000 + const form = decodePaymentPayload( + bytes(`__proto__=safe&constructor=value&${'x=1&'.repeat(count)}`), + 'application/x-www-form-urlencoded' + ) as Record + expect(Object.getPrototypeOf(form)).toBeNull() + expect(form.__proto__).toBe('safe') + expect(form.constructor).toBe('value') + expect(form.x).toHaveLength(count) + }) +}) diff --git a/packages/sdk/src/auth/utils/__tests/paymentTransport.test.ts b/packages/sdk/src/auth/utils/__tests/paymentTransport.test.ts new file mode 100644 index 000000000..08004b76f --- /dev/null +++ b/packages/sdk/src/auth/utils/__tests/paymentTransport.test.ts @@ -0,0 +1,83 @@ +import { + preparePaymentTransport, + authenticatedContentType, + buildMultipartPayment, + findPaymentBytes, + paymentBoundary, + paymentTransports, + resolvePaymentTransportLimits +} from '../paymentTransport.js' +import fc from 'fast-check' + +describe('BRC-118 transport selection primitives', () => { + it('preserves legacy content types and binds the exact multipart boundary', () => { + expect(authenticatedContentType('application/json; charset=utf-8')).toBe('application/json') + const multipart = 'multipart/form-data; boundary="ABC"' + expect(authenticatedContentType(multipart)).toBe(multipart) + expect(paymentBoundary(multipart)).toBe('ABC') + }) + it('treats absent capabilities as header-only and ignores unknown tokens', () => { + expect([...paymentTransports(null)]).toEqual(['header']) + expect([...paymentTransports('header, unknown, multipart, header')]).toEqual([ + 'header', + 'multipart' + ]) + expect([...paymentTransports('unknown')]).toEqual([]) + expect(() => paymentTransports('x'.repeat(257))).toThrow() + }) + it('validates configurable budgets before wallet work', () => { + expect(resolvePaymentTransportLimits().maxPaymentHeaderBytes).toBe(8192) + for (const maximum of [0, -1, Infinity, 1.5, 8193]) { + expect(() => resolvePaymentTransportLimits({ maxPaymentHeaderBytes: maximum })).toThrow() + } + }) + it('matches native byte search even for adversarial repeated prefixes', () => { + fc.assert( + fc.property( + fc.uint8Array({ maxLength: 1000 }), + fc.uint8Array({ minLength: 1, maxLength: 70 }), + (bytes, needle) => { + expect(findPaymentBytes(bytes, needle)).toBe(Buffer.from(bytes).indexOf(needle)) + } + ) + ) + expect( + findPaymentBytes( + new Uint8Array(100_000).fill(65), + new Uint8Array([...Array(69).fill(65), 66]) + ) + ).toBe(-1) + }) + it('bounds the final body including framing and rejects media-type injection', () => { + const built = buildMultipartPayment('{}', undefined, 1000, 'fixed') + expect(() => buildMultipartPayment('{}', undefined, built.body.length - 1, 'fixed')).toThrow( + 'body limit' + ) + expect(() => + buildMultipartPayment( + '{}', + { bytes: new Uint8Array(), contentType: 'text/plain\r\nInjected: true' }, + 1000 + ) + ).toThrow() + }) + it('selects exactly at the header threshold and enforces aggregate header limits', () => { + const original = { method: 'POST', headers: {} } + const transports = new Set(['header', 'multipart']) + const limits = resolvePaymentTransportLimits() + expect(preparePaymentTransport('x'.repeat(8192), original, transports, limits).transport).toBe( + 'header' + ) + expect(preparePaymentTransport('x'.repeat(8193), original, transports, limits).transport).toBe( + 'multipart' + ) + expect(() => + preparePaymentTransport( + '{}', + { ...original, headers: { 'x-large': 'x'.repeat(16384) } }, + transports, + limits + ) + ).toThrow('aggregate header budget') + }) +}) diff --git a/packages/sdk/src/auth/utils/decodePaymentPayload.ts b/packages/sdk/src/auth/utils/decodePaymentPayload.ts new file mode 100644 index 000000000..d526682f9 --- /dev/null +++ b/packages/sdk/src/auth/utils/decodePaymentPayload.ts @@ -0,0 +1,24 @@ +import { toUTF8Strict } from '../../primitives/utils.js' + +/** Decode an authenticated application payload while retaining its separately owned raw bytes. */ +export function decodePaymentPayload( + body: Uint8Array | undefined, + contentType: string | undefined +): unknown { + if (body === undefined) return undefined + const mediaType = contentType?.split(';')[0].trim().toLowerCase() + if (mediaType === 'application/json' || mediaType?.endsWith('+json') === true) + return JSON.parse(toUTF8Strict(body)) as unknown + if (mediaType?.startsWith('text/') === true) return toUTF8Strict(body) + if (mediaType === 'application/x-www-form-urlencoded') { + const values = Object.create(null) as Record + for (const [key, value] of new URLSearchParams(toUTF8Strict(body))) { + const prior = values[key] + if (prior === undefined) values[key] = value + else if (Array.isArray(prior)) prior.push(value) + else values[key] = [prior, value] + } + return values + } + return body +} diff --git a/packages/sdk/src/auth/utils/index.ts b/packages/sdk/src/auth/utils/index.ts index 5c0df10ce..a67501d99 100644 --- a/packages/sdk/src/auth/utils/index.ts +++ b/packages/sdk/src/auth/utils/index.ts @@ -2,3 +2,4 @@ export * from './verifyNonce.js' export * from './createNonce.js' export * from './getVerifiableCertificates.js' export * from './validateCertificates.js' +export * from './paymentTransport.js' diff --git a/packages/sdk/src/auth/utils/paymentTransport.ts b/packages/sdk/src/auth/utils/paymentTransport.ts new file mode 100644 index 000000000..c0f5056fc --- /dev/null +++ b/packages/sdk/src/auth/utils/paymentTransport.ts @@ -0,0 +1,259 @@ +import Random from '../../primitives/Random.js' +import { toArray, toHex } from '../../primitives/utils.js' + +export interface PaymentTransportLimits { + /** Selection threshold, not a guarantee about a particular proxy. Default 8 KiB. */ + maxPaymentHeaderBytes?: number + /** Includes a 4 KiB reserve for authentication and HTTP framing. Default 16 KiB. */ + maxRequestHeaderBytes?: number + /** Maximum serialized payment JSON. Default 4 MiB. */ + maxPaymentBytes?: number + /** Complete transmitted request body, including multipart framing. Default 7 MiB. */ + maxBodyBytes?: number +} + +export interface ResolvedPaymentTransportLimits { + maxPaymentHeaderBytes: number + maxRequestHeaderBytes: number + maxPaymentBytes: number + maxBodyBytes: number +} + +export type PaymentTransportErrorCode = + | 'ERR_PAYMENT_SIZE' + | 'ERR_PAYMENT_TRANSPORT' + | 'ERR_PAYMENT_REQUIREMENTS_CHANGED' + | 'ERR_PAYMENT_OUTCOME_UNKNOWN' + | 'ERR_PAYMENT_CANCELLED' + +/** Permanent transport refusals are not retried automatically. No payment bytes are attached. */ +export class PaymentTransportError extends Error { + readonly retryable = false + constructor( + readonly code: PaymentTransportErrorCode, + message: string, + readonly payment?: { + txid: string + state: 'prepared' | 'submitted' | 'uncertain' + aborted?: boolean + }, + readonly httpStatus?: number, + readonly authenticated?: boolean + ) { + super(message) + this.name = 'PaymentTransportError' + } +} + +export function resolvePaymentTransportLimits( + options: PaymentTransportLimits = {} +): ResolvedPaymentTransportLimits { + const limits = { + maxPaymentHeaderBytes: options.maxPaymentHeaderBytes ?? 8192, + maxRequestHeaderBytes: options.maxRequestHeaderBytes ?? 16 * 1024, + maxPaymentBytes: options.maxPaymentBytes ?? 4 * 1024 * 1024, + maxBodyBytes: options.maxBodyBytes ?? 7 * 1024 * 1024 + } + for (const [key, value] of Object.entries(limits)) { + if (!Number.isSafeInteger(value) || value < 1 || value > 16 * 1024 * 1024) { + throw new RangeError(`${key} must be an integer from 1 through 16777216`) + } + } + // The released authenticated transport has a fixed per-header ceiling. + if (limits.maxPaymentHeaderBytes > 8192) + throw new RangeError('maxPaymentHeaderBytes cannot exceed 8192') + return limits +} + +export function paymentTransports(advertisement: string | null): ReadonlySet { + if (advertisement === null) return new Set(['header']) + if (advertisement.length > 256) + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'Payment transport advertisement is too large.' + ) + const transports = new Set() + for (const part of advertisement.split(',')) { + const value = part.trim() + if (value === 'header' || value === 'multipart') transports.add(value) + } + return transports +} + +/** Keep released non-multipart preimages; bind the exact multipart boundary and parameters. */ +export function authenticatedContentType(value: string): string { + return isMultipartPaymentType(value) ? value : value.split(';')[0].trim() +} + +export function isMultipartPaymentType(value: string): boolean { + return value.split(';')[0].trim().toLowerCase() === 'multipart/form-data' +} + +export function paymentBoundary(contentType: string): string { + // One unambiguous boundary parameter; quoted MIME token boundaries are accepted. + const match = + /^multipart\/form-data\s*;\s*boundary=(?:([a-z0-9'()+_,./:=?-]{1,70})|"([a-z0-9'()+_,./:=?-]{1,70})")\s*$/i.exec( + contentType + ) + if (match == null) + throw new PaymentTransportError('ERR_PAYMENT_TRANSPORT', 'Invalid multipart payment boundary.') + return match[1] ?? match[2] +} + +export function paymentPayloadContentType(value: string): string { + if ( + value.length === 0 || + value.length > 1024 || + /[^\x20-\x7e]/.test(value) || + !/^[!#$%&'*+.^`|~\w-]+\/[!#$%&'*+.^`|~\w-]+(?:\s*;.*)?$/.test(value) + ) { + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'Invalid original payload Content-Type.' + ) + } + return value +} + +/** Linear-time byte search, also used to rule out delimiter collisions before signing. */ +export function findPaymentBytes(bytes: Uint8Array, needle: Uint8Array, from = 0): number { + const prefix = new Uint32Array(needle.length) + for (let index = 1, length = 0; index < needle.length;) { + if (needle[index] === needle[length]) prefix[index++] = ++length + else if (length > 0) length = prefix[length - 1] + else prefix[index++] = 0 + } + for (let index = from, length = 0; index < bytes.length;) { + if (bytes[index] === needle[length]) { + index++ + if (++length === needle.length) return index - length + } else if (length > 0) length = prefix[length - 1] + else index++ + } + return -1 +} + +export interface MultipartPaymentBody { + contentType: string + body: Uint8Array +} + +function paymentRequestHeaders(originalHeaders: Record): Record { + const headers: Record = Object.create(null) + for (const [name, value] of Object.entries(originalHeaders)) { + const lower = name.toLowerCase() + if (Object.hasOwn(headers, lower)) + throw new PaymentTransportError('ERR_PAYMENT_TRANSPORT', 'Duplicate request header.') + if (lower !== 'x-bsv-payment') headers[lower] = value + } + return headers +} + +export function preparePaymentTransport( + paymentJSON: string, + original: { method: string; headers: Record; body?: Uint8Array }, + transports: ReadonlySet, + limits: ResolvedPaymentTransportLimits +): { headers: Record; body?: Uint8Array; transport: 'header' | 'multipart' } { + const paymentSize = toArray(paymentJSON, 'utf8').length + if (paymentSize > limits.maxPaymentBytes) + throw new PaymentTransportError( + 'ERR_PAYMENT_SIZE', + 'Payment JSON exceeds its configured limit.' + ) + const headers = paymentRequestHeaders(original.headers) + const originalType = headers['content-type'] + let body = original.body + let transport: 'header' | 'multipart' = 'header' + if ( + paymentSize > limits.maxPaymentHeaderBytes || + !transports.has('header') || + (originalType !== undefined && isMultipartPaymentType(originalType)) + ) { + if (!transports.has('multipart')) + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'This payment requires multipart support from the authenticated server.' + ) + if (['GET', 'HEAD'].includes(original.method.toUpperCase())) + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'Fetch cannot carry a multipart payment with GET or HEAD; use a server-supported body-bearing route.' + ) + const multipart = buildMultipartPayment( + paymentJSON, + body === undefined + ? undefined + : { bytes: body, contentType: originalType ?? 'application/octet-stream' }, + limits.maxBodyBytes + ) + headers['content-type'] = multipart.contentType + body = multipart.body + transport = 'multipart' + } else headers['x-bsv-payment'] = paymentJSON + if (body !== undefined && body.length > limits.maxBodyBytes) + throw new PaymentTransportError('ERR_PAYMENT_SIZE', 'Paid request exceeds the body limit.') + let headerBytes = 4096 + for (const [name, value] of Object.entries(headers)) { + headerBytes += toArray(name, 'utf8').length + toArray(value, 'utf8').length + 4 + } + if (headerBytes > limits.maxRequestHeaderBytes) + throw new PaymentTransportError( + 'ERR_PAYMENT_SIZE', + 'Paid request exceeds the aggregate header budget.' + ) + return { headers, body, transport } +} + +/** Serialize once, then sign and transmit these same owned bytes. Never pass native FormData. */ +export function buildMultipartPayment( + paymentJSON: string, + payload: { bytes: Uint8Array; contentType: string } | undefined, + maximumBytes: number, + boundary = `----BsvPayment${toHex(Random(16))}` +): MultipartPaymentBody { + const contentType = `multipart/form-data; boundary=${boundary}` + paymentBoundary(contentType) + const utf8 = (text: string): Uint8Array => new Uint8Array(toArray(text, 'utf8')) + const payment = utf8(paymentJSON) + const delimiter = utf8(`--${boundary}`) + if ( + findPaymentBytes(payment, delimiter) !== -1 || + (payload !== undefined && findPaymentBytes(payload.bytes, delimiter) !== -1) + ) { + throw new PaymentTransportError( + 'ERR_PAYMENT_TRANSPORT', + 'Multipart boundary collides with payload.' + ) + } + const pieces = [ + utf8( + `--${boundary}\r\nContent-Disposition: form-data; name="x-bsv-payment"\r\nContent-Type: application/json\r\n\r\n` + ), + payment + ] + if (payload !== undefined) { + pieces.push( + utf8( + `\r\n--${boundary}\r\nContent-Disposition: form-data; name="body"\r\nContent-Type: ${paymentPayloadContentType(payload.contentType)}\r\n\r\n` + ), + payload.bytes + ) + } + pieces.push(utf8(`\r\n--${boundary}--\r\n`)) + let length = 0 + for (const piece of pieces) length += piece.length + if (!Number.isSafeInteger(maximumBytes) || maximumBytes < 1 || length > maximumBytes) { + throw new PaymentTransportError( + 'ERR_PAYMENT_SIZE', + 'Multipart payment exceeds the request body limit.' + ) + } + const body = new Uint8Array(length) + let offset = 0 + for (const piece of pieces) { + body.set(piece, offset) + offset += piece.length + } + return { contentType, body } +} diff --git a/packages/sdk/src/remittance/__tests/BasicBRC29.interop.test.ts b/packages/sdk/src/remittance/__tests/BasicBRC29.interop.test.ts new file mode 100644 index 000000000..8ce7e4e6d --- /dev/null +++ b/packages/sdk/src/remittance/__tests/BasicBRC29.interop.test.ts @@ -0,0 +1,63 @@ +import { Brc29RemittanceModule } from '../modules/BasicBRC29.js' +import { ProtoWallet } from '../../wallet/ProtoWallet.js' +import type { WalletInterface } from '../../wallet/Wallet.interfaces.js' +import PrivateKey from '../../primitives/PrivateKey.js' +import PublicKey from '../../primitives/PublicKey.js' +import Transaction from '../../transaction/Transaction.js' +import Script from '../../script/Script.js' +import P2PKH from '../../script/templates/P2PKH.js' + +const payerKey = new PrivateKey(71) +const recipientKey = new PrivateKey(72) +const payer = new ProtoWallet(payerKey) +const recipient = new ProtoWallet(recipientKey) +const protocolID: [2, string] = [2, '3241645161d8'] +const prefix = 'cHJlZml4' +const suffix = 'c3VmZml4' + +async function artifact(payToSender: boolean) { + const key = await payer.getPublicKey({ + protocolID, + keyID: `${prefix} ${suffix}`, + counterparty: recipientKey.toPublicKey().toString(), + forSelf: payToSender + }) + const source = new Transaction() + source.addOutput({ satoshis: 1000, lockingScript: Script.fromASM('OP_TRUE') }) + const tx = new Transaction() + tx.addInput({ + sourceTransaction: source, + sourceOutputIndex: 0, + unlockingScript: Script.fromASM('OP_TRUE') + }) + tx.addOutput({ + satoshis: 100, + lockingScript: new P2PKH().lock(PublicKey.fromString(key.publicKey).toAddress()) + }) + return { + customInstructions: { derivationPrefix: prefix, derivationSuffix: suffix }, + transaction: tx.toAtomicBEEF(), + amountSatoshis: 100, + outputIndex: 0 + } +} + +describe('BRC-29 independent payer and recipient key derivation', () => { + it.each([false, true])( + 'accepts only a recipient-spendable output; pays sender=%s', + async paysSender => { + const wallet = recipient as unknown as WalletInterface + wallet.internalizeAction = jest.fn(async () => ({ accepted: true })) + const result = await new Brc29RemittanceModule().acceptSettlement( + { + threadId: 'two-wallet-interop', + sender: payerKey.toPublicKey().toString(), + settlement: await artifact(paysSender) + }, + { wallet, now: () => 1 } + ) + expect(result.action).toBe(paysSender ? 'terminate' : 'accept') + expect(wallet.internalizeAction).toHaveBeenCalledTimes(paysSender ? 0 : 1) + } + ) +}) diff --git a/packages/wallet/wallet-toolbox/CHANGELOG.md b/packages/wallet/wallet-toolbox/CHANGELOG.md index f1933a15d..9cfdfe6a3 100644 --- a/packages/wallet/wallet-toolbox/CHANGELOG.md +++ b/packages/wallet/wallet-toolbox/CHANGELOG.md @@ -4,6 +4,192 @@ This document captures the history of significant changes to the wallet-toolbox The git commit history contains the details but is unable to draw attention to changes that materially alter behavior or extend functionality. +## 2.15.0 candidate — bounded synchronization and canonical proof recovery + +- Preserve queued primary reselection: a request to select A during a pending + switch from A to B waits for that switch and then selects A, including after + an earlier failure. Check no-op selections under the existing ownership + boundary and preserve valid sync sessions when a progress formatter throws + before the transition. After partial propagation failure, reload persisted + selections before authorizing or executing queued access. Conflicting stores + refuse active authorization; failed reloads release ownership for retry. No + API, wire or schema migration is required. + +- Add internal monotonic continuity-floor transactions and bounded primary-key + tombstone collection. Current receipt locks pin all live prefixes; collection + preserves live/newer records and all thirteen source tables. WAL/RC/RR fixtures + cover partial deletion, rollback and committed lost acknowledgements. Provider + maintenance integration, runtime quotas, registered recovery and full #544 + acceptance remain required; incremental reader advertisement stays off. + +- Add internal owned journal capture for `better-sqlite3` WAL and static `mysql2`. + Reserve both pools before the short writer barrier; pin generation/profile, + commit the exact receipt and verify closure before publication. Retain source + admission through physical cleanup and fence an unproved close. Native fixtures + cover commit/publication process loss, immutable pages and writer progress. + Provider retention ownership, quotas, delta receiver integration and full #544 acceptance + remain unfinished; no migration or incremental capability is advertised. + +- Preserve main's Postgres storage and legacy sync support. SQLite/MySQL snapshot + auxiliary migrations are recorded as no-ops there, with their capabilities + unavailable. Future Postgres snapshot support requires new forward migrations. + +- Add internal, unadvertised SQL journal foundation: exact revisions, bounded + pages/bootstrap, full-table observers and source-bound generation ownership + with interrupted-installation recovery. No public capability, registered + migration, deployment or completed incremental sync is introduced. Full + quota/receiver/primary and portability acceptance remains pending. + +- Repair SQLite replacement maintenance with a new auxiliary generation, durable + displaced-owner witnesses, bounded resumable source copying and physical-row + retirement. Preserve source tables, profile/reference semantics and pinned + ordinary/archive views. Bind legacy trigger ownership to its object type so + foreign views sharing a trigger name cannot bypass retirement refusal. Apply + the explicit forward migration; normal downgrade + refuses, while explicit full-data deletion remains supported. Complete native, + mutation, larger-wallet and hosted acceptance remain required. + +- Accept a valid all-zero terminal offset reset when an unchanged backup keeps + the same timestamp. Preserve remote state binding, monotonic timestamps and + nonterminal/partial-reset checks in HTTP and resumable sessions. Repeated + ordinary/binary HTTP backups and lost terminal acknowledgements are covered; + returned provider failures retain precedence over checkpoint fields. Checkpoint + bytes and stored schemas remain unchanged. + +- Add resumable global proof/request ownership indexes with exact reference + counts and proof-presence guards. Preserve both ownership bases, current + independent-writer changes and pinned ordinary/archive views. Bootstrap, + process-loss recovery and native first/late page qualification keep standard + rows/indexes and portable/cursor bytes unchanged. Complete system acceptance + and the remaining sync/portability program are still required. + +- Add resumable certificate-field profile indexes preserving source collation, + empty names, direct/parent ownership and exact rename bytes. Ordinary and + archive readers adopt complete migration state inside their retained view; + standard indexes, legacy OFFSET order and BRC-38/cursor bytes stay unchanged. + The subsequent global checkpoint extends this selection to proofs/requests; + the full program remains incomplete. + +- Add auxiliary numeric relationship indexes for label/tag maps without changing + standard indexes, composite cursors, legacy OFFSET order or BRC-38 bytes. + Preserve both parent ownership bases through moves, rekeys, tombstones and + deletion; retain inconsistent relationships for explicit closure refusal. + Resume bounded bootstrap after interrupted DDL/transactions and use the + matching auxiliary primary index, recorded membership and indexed source + lookups for bounded MySQL pages before and after statistics refresh. The remaining indirect + tables and full sync/portability program are incomplete. + +- Add an exact-claim source-owner fence and additive owner migration. Remote + cancellation cannot release archive/request capacity or publish ready before + the owning source and pool have closed. Append checks cancellation atomically; + direct archive cleanup follows the same fence. The reader validates request-bound + pending-cleanup receipts and uses a separate fixed deadline, bounded backoff and + awaited I/O settlement. Add fixed-slot SQLite WAL/MySQL backend guards and + exact-owner recovery only after physical connection closure. Preserve old + unguarded reservations and refuse changed backend identities. Reader + advertisement stays disabled pending complete qualification. + Keep the provider's source slot fenced after unproved native cleanup, including + an already-settled read promise; preserve retry after an ordinary read failure + whose physical cleanup succeeds. + +- Add the unadvertised remote row-reader foundation: immutable server-issued + offers, exact-request retry, fixed leases, verified packed rows and durable + cursor integration with local sync. Reader advertisement stays disabled while + cross-replica physical cleanup and owner recovery remain incomplete. + +- Preserve the public hash-wasm Argon2id generic signature in emitted types. + Strict CommonJS and ESM require-consumer checks cover binary/string results + without ambient internal-bundle declarations. Runtime bytes and SDK peer ranges + remain unchanged. + +- Add shared SQL snapshot staging with profile-bound internal capture ownership, + immutable completed pages, exact replay receipts, explicit logical byte/page + reservations and resumable bounded cleanup. A second auxiliary migration + leaves standard tables, legacy sync checkpoints and wire formats unchanged. + The local capture controller binds source metadata/schema to one read view, + verifies profile relationships and stores all thirteen raw tables through + bounded binary frames, with cancellation and cleanup. Canonical portable + validation and larger-wallet policy remain open. + +- Add negotiated authenticated archive admission/status/directory/page/cancel + methods for migrated static WAL/MySQL sources, with exact profile-bound + arguments and no writer credentials. Both client variants enforce a separate + two-MiB response ceiling and retain server/storage identity binding. Expired + request cleanup precedes admission; resource-limit receipts are explicit. + Shutdown awaits physical capture and HTTP cleanup. Remote row-reader/manager + integration, portable semantics and the full program remain incomplete. + +- Integrate coherent SQL pages into ordinary local push, pull and backup, with + a dedicated source reader and short, fair destination commits. Add resumable + push progress/cancellation. Commit rows, normalized ID maps and durable cursors + together; reject stale sessions and independent primary ABA transitions. + Preserve source primary metadata for push/backup and destination selection + for pull, including serialized fallback when the manager's cache is older. + Add the version-one auxiliary schema migration and primary-epoch trigger. + Existing legacy checkpoint JSON is preserved. Unsupported configurations and + explicit row/reference/retention limits retain serialized fallback; genuine + validation and I/O errors reject. `snapshotSync: false` is the supported forward + rollback with schema retained. Primary reconciliation, remote/IDB retained + views, streaming and staged restore remain required work. + +- Add local profile-bound SQL keyset pages over retained views. All thirteen + standard tables preserve tombstones and original source records; binary values + stay packed. SQL preflights bounded keys and payload sizes before fetching a + page. Oversized individual rows explicitly refuse pending large-value streaming. + Cursors belong only to their live view and table; expiry/process loss requires + restart. The page primitive preserves existing indexes, OFFSET checkpoints and RPC. + This is a prerequisite for the active streaming/resumable-backup program. + +- Add local retained SQLite/MySQL views with explicit lifetime, cancellation and + cleanup ownership. Opening pins the transaction before the first consumer read; + one view per provider and one read per view prevent unbounded admission. Closing + discards late results and awaits physical cleanup, including failed reads. + Internal opening/cleanup rejections remain observed if a view is cancelled + before its opening consumer resumes; callers still receive the original errors. + Driver/query deadlines remain separate. Each view occupies a pool connection; + IndexedDB and RPC explicitly remain unsupported for retention. This is a paging + prerequisite; the separate sync integration above changes scheduling and adds + auxiliary persistence without changing wire formats. + +- Capture BRC-38 source metadata and every table from one provider read view. + Custom providers opt in with `supportsReadSnapshot` and `readSnapshot`. The + additive `requireSnapshot` export option refuses unsupported views; old + custom-provider calls keep their caller-quiesced compatibility path. SQLite, + MySQL and IndexedDB implement the local boundary. MySQL uses valid + next-transaction characteristics on one reserved connection and closes failed + connections before returning them to the pool. Recognized optional nullable + JSON object fields normalize in a detached copy without + dropping array values. Streaming files and the full #544 program remain in + implementation; IndexedDB writers still wait during source capture. +- Add resumable local atomic pages, durable checkpoints, cancellation/progress, + concurrent read capability checks and fair foreground/background ownership. +- Separate fixed source/commit latency from marginal row cost and bound upward + probes, page records and proof concurrency. +- Reduce redundant storage promise forwarding while retaining authorization + inside exclusive ownership; failures release the next queued operation. +- Share the remote forwarding rejection boundary, skip uncontended priority + searches and avoid repeated queue/checkpoint helper allocations. Custom RPC + throws remain Promise rejections; wire, result and fairness contracts remain. +- Share reader/writer admission helpers without moving authorization across the + queue boundary. Normalize entity arrays in place using one field list and a + Map of property descriptors, avoiding an intermediate mapping array + while preserving date, null, byte and non-enumerable-field behavior. +- Adjust only the reviewed mobile Hermes Brotli ceiling to 1,675,000 bytes, + retaining the other five limits. The mobile README records composition, + upstream comparison and 9.97% measured headroom; build settings are unchanged. +- Check chains before sync writes and preserve returned, serialized and thrown + failures without advancing progress. +- Repair stale selected/input/broadcast proofs against canonical evidence; fence + monitor updates against primary replacement and concurrent proof corrections. +- Keep proof changes and prepared-BEEF invalidation atomic, and retain safe custom + provider behavior. No persisted-schema migration is required. +- Use lazy asynchronous producers for sequential storage and proof work, preserving the + eight-worker proof limit, failure draining, deterministic transaction order + and Promise rejection contracts. +- Exercise large copies, tombstones, restart/lost acknowledgements and foreground + latency on SQLite, authenticated HTTP and native Chromium IndexedDB. Inclusive + timestamp boundary traffic remains a snapshot/high-water follow-up. + ## wallet-toolbox 2.14.5 - `WalletPermissionsManager` retires no-send transaction ownership and reference diff --git a/packages/wallet/wallet-toolbox/README.md b/packages/wallet/wallet-toolbox/README.md index 787a1c2a4..c4a32143b 100644 --- a/packages/wallet/wallet-toolbox/README.md +++ b/packages/wallet/wallet-toolbox/README.md @@ -6,6 +6,12 @@ A [BRC-100](https://github.com/bitcoin-sv/BRCs/blob/master/wallet/0100.md) conforming wallet implementation for the BSV blockchain, built on the [BSV SDK](https://bsv-blockchain.github.io/ts-stack/packages/sdk/). Provides persistent storage, protocol-based key derivation, transaction monitoring, chain tracking, and signing — everything needed to build wallet-powered applications on BSV. +Postgres storage retains standard wallet migrations and legacy synchronization. +The candidate's SQLite/MySQL auxiliary snapshot migrations are recorded as no-ops +on Postgres, and its retained, paged and incremental snapshot capabilities are +unavailable there. Future Postgres snapshot support requires forward migrations; +those no-op entries must not be rewritten as deployed schema changes. + ## Backup and recovery: keep both keys and wallet data **A root key or seed alone is not a complete BRC-100 wallet backup.** Users @@ -28,6 +34,159 @@ These guides cover the recovery design. The measurements below describe specific tests and do not establish that every wallet product has complete key-and-data disaster recovery. +The unpublished 2.15 candidate captures BRC-38 source settings, wallet identity +and standard table closure in one local provider read view. SQLite and IndexedDB +tests cover independent writes during capture. A local MySQL 8.4.11 fixture +verifies repeatable-read isolation, read-only enforcement and connection cleanup +without changing session defaults. Exact proof and checkpoint reads inside these +provider-owned snapshots retain the consistent view without requesting write +locks; ordinary writable transactions retain their row locks. The critical +`wallet-read-snapshot-consistency` mutation target covers the complete transaction +classification and lookup methods with all six retained, portable and generated +read-view suites. Generated schedules exercise direct/retained failure cleanup, +cross-provider tokens, partial/exact selectors and independent writable work; +ordinary lookups also preserve returned checkpoint date/boolean normalization. +The original property strength and zero-uncovered/invalid/unexecuted mutation +gates apply. Deployed/PXC recovery remains unqualified. Custom providers +opt in with `supportsReadSnapshot` and `readSnapshot`. Use the export option +`requireSnapshot: true` to refuse unsupported capture; old custom-provider calls +retain their documented caller-quiesced fallback. +Recognized optional nullable JSON fields are omitted in a detached archive copy; +array entries and meaningful falsy values are preserved. The helpers still +materialize the full document/file, and IndexedDB writers wait during capture. +Run `pnpm test:snapshot-archive-crash` for native SQLite process recovery and +`pnpm test:snapshot-archive-mysql` for the disposable MySQL fixture from this +package. Local MySQL qualification uses Docker Desktop and requires the pinned +image already present. Required wallet CI shard 1 provisions that same immutable +image on its ephemeral Linux runner, then runs both native fixtures from the +same-head build before Jest. Hosted MySQL mode accepts only the local default +Unix socket. The fixture uses a unique ownership label, loopback port, bounded +memory/CPU/process count and temporary data volume; individual Docker calls, +readiness and the child proof have deadlines. Failure or cancellation drains +owned work and attempts exact-owner cleanup before reporting its outcome; +unproved cleanup fails qualification. Other wallet shards do not start MySQL. + +The candidate also contains internal SQL journal primitives for exact revisions, +bounded metadata pages, bootstrap with a durable explicit row allowance, +source-table observers, and recovery of an +interrupted journal installation. Owned generations also bind explicit receipt +capacity/lifetime policy; exact receipts are persisted, read and collected in +bounded caller-owned transactions. Receipts prove a captured prefix and cannot +reopen a lost retained view. The internal `openSnapshotJournalSource` controller +reserves two owned native connections before its writer barrier, pins the profile +and generation, persists the receipt, and publishes after receipt commit and +closure verification. It requires an existing file-backed WAL database with +`better-sqlite3` or a static `mysql2` connection; other drivers refuse before pool +construction. Physical cleanup retains provider admission, and an unproved close +fences further sources. If source work and transaction cleanup both fail, the +ownership error retains the original source failure and each rollback cause. +Run `pnpm test:snapshot-journal-crash` for the +SQLite process-loss fixture and `pnpm test:snapshot-journal-mysql` for the isolated +MySQL client/server-process recovery fixtures. Internal floor transactions reserve +the global writer clock before current receipt locks and cannot pass a live +receipt's prefix. Tombstone collection examines at most 256 primary-key rows, +including live and newer records, and binds its cursor to one epoch, floor and +stream. Native WAL/RC/RR checks cover twenty-four real process-loss boundaries. +The internal `maintainSnapshotJournal` provider controller validates the complete +owned generation and excludes the configured migration owner before each short +floor or collection transaction. It shares source admission through cancellation, +rollback and physical cleanup; failed cleanup fences further retained admission. +Automatic scheduling remains unfinished. +These helpers do not register a migration or advertise incremental +synchronization. Runtime quotas, registered recovery, generation-aware delta pages +and receiver/primary integration and the remaining issue #544 acceptance work are +still required; see the +[journal foundation](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#internal-journal-foundation-unadvertised). + +Required wallet CI shard 1 also runs both native journal entry points from the +same-head build after the existing archive fixtures. The complete journal mutation +aggregate and native recovery checks are separate qualification requirements. + +Both native archive entry points include interrupted auxiliary profile and numeric +relation migrations through the real migrator. The numeric relation fixture +terminates seven migration boundaries, repairs the abandoned migration lock and +compares recovered membership with an independent source-table oracle. The MySQL +entry point also observes relation writer/reader lock ordering under both +supported isolation levels and checks late composite pages against 8,192 +interleaved map rows per relation. It requires bounded handler reads and the +auxiliary primary-key range plan. These synthetic fixture results qualify the +tested engine/configuration; deployed PXC remains a separate acceptance gate. + +SQL providers also expose `supportsRetainedReadSnapshot` / `openReadSnapshot` +for a local view held across idle reads, with one view per provider, one read at +a time, and bounded lifetime/cancellation. Await `closed`/`close()` for physical +cleanup. This low-level view occupies a caller-pool connection; driver deadlines +remain separate and a single-connection pool cannot serve other work until +release. The [durable local SQL backup integration](#durable-local-sql-backup-integration-215-candidate) +below adds bounded pages and uses a dedicated reader to preserve foreground +capacity. IndexedDB and remote RPC do not expose retained views. See the +[retained view contract](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#retained-local-sql-read-views-unpublished-candidate). +The SQL candidate also includes [bounded archive staging](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#shared-sql-snapshot-staging-unpublished-internal-component) +for the ongoing remote snapshot implementation. This internal component does not +add an authenticated export endpoint or a browser/mobile database adapter. +Its metadata-only receipt directory binds every page and all thirteen table +positions to the archive root before arbitrary-table reads; each payload is +checked against that verified directory. The original binding JSON bytes are +preserved as the hash preimage. This verifies transport integrity, not the source's +honesty or portable transaction/proof semantics. +An additional internal SQL request table binds retries to an immutable deadline +and archive, reserves capacity before source acquisition, and retains bounded +terminal receipts through cleanup. Its internal capture controller shares the +provider's single owned reader slot, publishes readiness after physical cleanup, +and drains cancellation/shutdown. Completed archives survive controller replacement; +failed cleanup fences admission and retains its reservation. The candidate now +adds a negotiated authenticated archive transport on migrated WAL/MySQL servers +and both client variants, with bounded responses and durable admission/status. +`getSnapshotArchiveTransport(identityKey)` is a low-level API; remote row reading, +ordinary sync/export adoption and portable validation remain incomplete. Server +or client `snapshotArchives: false`, or provider `snapshotSync: false`, disables +this capability. See the [transport contract](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#authenticated-snapshot-archive-transport-unpublished-candidate). +The candidate's additive source-owner table now retains request/archive capacity +until the exact controller acknowledges physical source cleanup. Cross-controller +cancellation fences the next page; pending cleanup reports an error and keeps its +reservation. Do not mix older candidate binaries or remove the owner schema while +captures remain. The reader now waits for an exact completion acknowledgement through bounded +pending-cleanup polling, preserving independent failures. The additive guard +migration now permits exact-owner recovery after physical source closure, using +stable local SQLite WAL guard files or a same-server MySQL connection lock. +Older unguarded owners remain reserved, and changed backend identities fail closed. +An unproved native close also fences further snapshot sources on that provider, +even after its read promise settles. Ordinary read failures remain retryable +after physical cleanup succeeds. +Keep the guard files and bindings intact and drain captures before downgrade. +The server reader capability remains unadvertised pending complete qualification. +The auxiliary profile-key migration preserves standard-table indexes and legacy +OFFSET ordering while adding indexed snapshot selection for eight direct tables. +Its triggers track independent writers; bounded bootstrap batches survive restart, +and readers enable the index only from a complete migration in their retained +view. See the [migration and recovery contract](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#auxiliary-profile-indexes-unpublished-candidate). +The separate numeric relation migration indexes label/tag maps while retaining +both parent ownership bases, tombstones and cross-profile inconsistency checks. +It preserves composite cursor and legacy OFFSET order, resumes bounded bootstrap +after interruption, and bounds MySQL paging before and after statistics refresh +with the auxiliary primary index and indexed source lookups. See its +[migration and recovery contract](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#auxiliary-numeric-relation-indexes-unpublished-candidate). +The certificate-field migration adds collation-preserving composite keys and +independent direct/parent ownership. Empty text keys, interrupted bootstrap and +case-only renames retain their source meaning. It leaves standard indexes and +legacy cursor order intact; readers require the complete migration in their +pinned view. See the [certificate migration and recovery contract](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#auxiliary-certificate-field-indexes-unpublished-candidate). +The global proof/request migration adds exact profile reference counts and +presence guards, with resumable bootstrap and retained ordinary/archive reads. +Direct and request-based proof references remain independent; removing the last +reference collects its auxiliary presence guard. Standard rows, text comparisons, +indexes and cursor/portable bytes remain unchanged. See the +[global migration and recovery contract](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#auxiliary-global-proofrequest-indexes-unpublished-candidate). +The additive SQLite conflict-repair migration installs metadata-bound witnesses, +rebuilds all four index families in bounded resumable pages, and retires obsolete +auxiliary rows without changing standard wallet data. Reader adoption follows +journal publication inside the pinned view. Legacy ownership checks distinguish +triggers from views even when their names coincide; foreign readers of auxiliary +data refuse migration. Preserve partial state for recovery; +ordinary downgrade refuses, while explicit `dropAllData()` retains its destructive +contract. See the [SQLite generation migration contract](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#sqlite-conflict-safe-index-generation-unpublished-candidate). +The complete sync/streaming/restore program remains in progress on #569. + ## Backup and sync: tested results **Live E2E testing used a large wallet in the native desktop client**, covering @@ -51,14 +210,18 @@ Timing compares successive candidates, not a controlled comparison against upstr ### SQLite migration recovery -SQLite migration handling introduced in 2.13.2 runs migration DDL and the migration -journal update transactionally. Foreign-key enforcement is disabled before the +SQLite migration handling introduced in 2.13.2 runs transactional migration DDL +and the migration journal update together. The unpublished profile-index and +numeric relation, certificate-field and global proof/request migrations are explicit resumable exceptions: auxiliary keys and progress commit +in bounded batches before its final migration journal entry. Foreign-key enforcement is disabled before the migration transaction for table rebuilds and restored after success or failure. -Failed migrations can be retried after reopening the database without partial -schema objects from that attempt. MySQL's existing transaction configuration +Failed transactional migrations can be retried after reopening the database +without partial schema objects from that attempt. These resumable index +migrations instead retain their verified auxiliary objects and committed progress; +see its linked recovery contract before retrying an interrupted migrator. MySQL's existing transaction configuration is unchanged. -This prevents future partial migrations. It does not automatically repair a +Transactional migration handling prevents partial table rebuilds. It does not automatically repair a store already left with unjournaled schema objects by an older version. Preserve the database and verified backups and reconcile the exact schema and migration journal before recovery; do not delete journal rows or wallet data blindly. @@ -221,8 +384,13 @@ The toolbox publishes three npm packages from this repo: ### Sync performance and recovery Sync pages start at 64 records and adapt after successful commits toward a -five-second page budget. Proof-bearing pages cap growth at 128 records; cheap -metadata pages can grow to 1,000, while provider byte/item ceilings still apply. +five-second marginal-work budget. A bounded history separates fixed read and +commit overhead from per-record work, so slow fixed latency does not collapse +large copies to one record per request. A two-record probe permits recovery from +the single-record floor. Proof-bearing pages cap growth at 128 records; metadata +pages can grow to 1,000, while provider byte/item ceilings still apply. These are +work estimates, not deadlines: an individual proof or unavailable dependency can +still take longer. The server checks at most eight proofs concurrently and waits for all started checks to settle on failure before rejecting the page. Every proof still passes transaction, Merkle path, active-root and active-header validation before a merge. @@ -245,9 +413,9 @@ client bundle cost. The [artifact measurements and limits](./docs/sync-transfer. include the combined upstream security fixes. These are explicit feature costs; the RPC validation coordinator remains excluded from browser/mobile bundles. -The transfer extension is an **unpublished 2.13.0 candidate**. Published 2.12.0 -has no record-transfer methods. Check exact build provenance and authenticated -runtime capabilities, not a version label alone. An oversized record on a legacy +The transfer extension is included in the published 2.13.2 graph. Older 2.12.0 +providers have no record-transfer methods. Check exact build provenance and +authenticated runtime capabilities, not a version label alone. An oversized record on a legacy source cannot be rescued by upgrading only its destination; upgrade the source before retrying. Records exceeding the negotiated 64 MiB frame limit fail safely without being skipped or advancing their checkpoint. @@ -372,6 +540,94 @@ retained privately, outside this repository. page for both Knex and IndexedDB storage, including short final pages and pages requested at or past the end of the result set. +### Resumable pulls and foreground access + +`WalletStorageManager.syncFromReaderResumable(identityKey, reader, options)` +adds per-page progress and cancellation without changing the existing sync +method signatures. The result reports `completed` or `cancelled`, page and row +counts, the last acknowledged checkpoint, and the selected execution mode. + +The new resumable API defaults to a 256 KiB rough page target, with an explicit +ceiling up to 10 MB. Existing sync methods retain their previous defaults. + +```ts +const cancellation = new AbortController() +const result = await storage.syncFromReaderResumable(identityKey, reader, { + signal: cancellation.signal, + maxItems: 128, + maxRoughSize: 262144, + onProgress: progress => console.log(progress.state, progress.pages) +}) +// A later invocation loads durable destination progress, including any page +// whose acknowledgement was lost. Do not replay a saved request manually. +``` + +Local SQLite/MySQL and IndexedDB destinations advertise `storageAccess.version` +1 with atomic sync checkpoints. Their paged pull reads and prepares one source +page outside manager write ownership, then queues its atomic data/checkpoint +commit. Network-backed proof checks finish before that queue is acquired; the +commit rejects changed proof records. Payloads are detached during preparation, +and a prepared page can only be consumed once. Foreground operations can run +between pages and during source/proof I/O. Supported reads share up to eight +slots; writers remain exclusive. A waiting background page gets a turn within +eight foreground grants or after one second of waiting at the next release. +This bounds queue preference, not the duration of a provider operation. + +Cancellation before a commit discards that page. Cancellation during a commit +waits for its acknowledgement, reports the committed checkpoint, then stops. +Source I/O also settles before the stopped result: custom providers must supply +their own I/O deadlines. Failures propagate without blind write replay; restart +loads the durable checkpoint. Changing the selected primary fences an older +session before its next write. Concurrent copies of the same source cannot both +commit the same checkpoint. Progress observers receive independent checkpoint +copies and run outside page ownership in paged mode. + +Missing capabilities, remote destinations, self-copies and primary reconciliation +retain exclusive execution. Supported ordinary local SQL copies use the durable +snapshot path described below. The legacy atomic-page capability probe retains +its serialized fallback; snapshot validation and I/O failures reject. Page ceilings are rough encoded +size hints; the existing negotiated per-record transfer bounds still apply to +large records. One page is in flight, and progress does not accumulate wallet +records or per-record logs. Legacy source pagination retains the existing eventual +replication contract. Supported local SQL sources use one coherent retained view; +the destination remains a merged replica. Remote snapshot handles and streaming +portable archives require their separate consistency and format contracts. +The existing timestamp boundary is inclusive: an unchanged copy can reread rows +sharing the final timestamp, including an entire same-timestamp import. Those +rows are not rewritten. This protects late same-time arrivals; a coherent source +revision/snapshot is needed to eliminate that boundary traffic safely. Smaller +`maxRoughSize` values (for example 262144) reduce transient authenticated HTTP +memory and event-loop work for constrained devices, at the cost of more pages. + +Every live sync checks declared network chains before registering a destination +user or checkpoint. A missing or unrecognized chain is not inferred. Thrown and +returned provider errors take precedence over `done`, counters and checkpoint +hints; unfinished pages without durable progress stop instead of spinning. +`WERR_NETWORK_CHAIN` and `ProcessSyncChunkResult.error` remain compatible. + +### Canonical proof recovery during actions + +When services are configured, selected-change BEEF is checked before returning +to the signer, including embedded ancestry and prepared artifacts. The actual +send/monitor path checks its rebuilt bundle as well. Valid graphs avoid extra +proof-record reads and copies; checks run with at most eight operations in +flight. Stale roots trigger bounded canonical lookups. Replacement transaction +bytes, Merkle membership, root and active header must agree before use. + +SQLite/MySQL and IndexedDB persist verified corrections with a compare-and-set +against the original proof, and invalidate prepared artifacts in the same +transaction. A concurrent repair is never overwritten. Custom providers may +implement `compareAndSetProvenTxProof`; the default repairs only the outgoing +graph. Failed recovery retains the source records, stops before broadcast, and +returns `WERR_INVALID_MERKLE_ROOT` with its txid, root and height across JSON-RPC. +Failed construction releases its funding reservation through the existing +failed-action cleanup; it does not mark the input spent without evidence. + +Standalone storage construction without configured services retains its offline +contract; its caller must validate before signing or broadcasting. This change +does not audit historical block-hash metadata or repair every stored proof: +BEEF root validity and an operator's historical-store audit remain distinct. + ### UMP account continuity and phone changes Argon2id password derivation uses a proven-ready host backend when one is @@ -1122,3 +1378,36 @@ for the full stack-wide policy. This package is released under the [Open BSV License Version 6](./LICENSE.txt). The accompanying [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md) and [LICENSES/](./LICENSES/) preserve the package's earlier Open BSV grant. + +### Local packed snapshot pages (2.15 candidate) + +SQL providers expose `supportsWalletReadSnapshot()` and +`openWalletReadSnapshot(identityKey, options)`. Settings, the selected user and +all thirteen standard tables share one retained read view. `readPage(table, +cursor, { maxRows, maxBytes })` uses stable storage keys and checks payload size +before loading rows; binary columns remain `Uint8Array`. Always close the view. +Cursors expire with the view and cannot resume after process loss. Oversized rows +explicitly refuse pending large-value streaming. IndexedDB and RPC do not expose +this capability; the page primitive preserves existing indexes and legacy cursors. See the +[page contract and limits](../../../docs/guides/wallet-sync-reliability.md#profile-bound-local-sql-pages). +The full #544 program remains incomplete. + +### Durable local SQL backup integration (2.15 candidate) + +Supported ordinary push, pull and backup calls now consume those pages outside +manager ownership and commit each destination page, ID mapping and cursor +atomically. `syncToWriterResumable` joins `syncFromReaderResumable` for progress and +cancellation. A dedicated reader preserves foreground pool capacity. SQLite +requires file-backed WAL; MySQL requires a static database connection. Unsupported +providers, oversized rows and retention limits use the serialized fallback. +Push and backup propagate the source view's stored primary selection, even when +the manager's cached selection is older. Pull keeps its destination selection. +The same direction-specific behavior applies during serialized fallback. + +Apply migration `2026-09-30-001 add durable snapshot sync` through `migrate()`. +It adds auxiliary session/mapping/primary-epoch tables and a primary-change trigger; +legacy checkpoint JSON and standard-table indexes are preserved. Use +`snapshotSync: false` for forward rollback on a current binary while keeping the +schema. See [durability, limits and downgrade guidance](../../../docs/guides/wallet-sync-reliability.md#durable-local-sql-sync-and-ordinary-backup). +Primary reconciliation, remote/IDB retained views, large-value streaming and +staged archive restore remain part of the incomplete #544 program. diff --git a/packages/wallet/wallet-toolbox/benchmarks/create-action-beef.bench.test.ts b/packages/wallet/wallet-toolbox/benchmarks/create-action-beef.bench.test.ts index 39c4eec2d..fe2133d8d 100644 --- a/packages/wallet/wallet-toolbox/benchmarks/create-action-beef.bench.test.ts +++ b/packages/wallet/wallet-toolbox/benchmarks/create-action-beef.bench.test.ts @@ -6,12 +6,7 @@ import { KnexSessionManager } from '../src/storage/remoting/KnexSessionManager' import { StorageServer, WalletStorageServerOptions } from '../src/storage/remoting/StorageServer' import { StorageKnex } from '../src/storage/StorageKnex' import { managedChangeOutputFields } from '../src/storage/methods/managedChange' -import { - TableOutput, - TableOutputBasket, - TableProvenTx, - TableTransaction -} from '../src/storage/schema/tables' +import { TableOutput, TableOutputBasket, TableProvenTx, TableTransaction } from '../src/storage/schema/tables' import { ScriptTemplateBRC29 } from '../src/utility/ScriptTemplateBRC29' import { BdkVerifier } from '@bsv/verifast' @@ -44,17 +39,14 @@ interface BenchmarkContext { ctx: TestWalletNoSetup basket: TableOutputBasket storageEvents: TelemetryEvent[] + canonicalRoots: Set } interface QueryProbe { - stop: () => { queryCount: number, databaseTransactions: number, databaseMs: number } + stop: () => { queryCount: number; databaseTransactions: number; databaseMs: number } } -function makeSourceTransaction ( - index: number, - satoshis: number, - lockingScript: Script -): Transaction { +function makeSourceTransaction(index: number, satoshis: number, lockingScript: Script): Transaction { const transaction = new Transaction() transaction.addInput({ sourceTXID: '00'.repeat(32), @@ -66,14 +58,11 @@ function makeSourceTransaction ( return transaction } -function makeBenchmarkMerklePaths (txids: string[], height: number, seed: number): MerklePath[] { +function makeBenchmarkMerklePaths(txids: string[], height: number, seed: number): MerklePath[] { const sharedLevels = Math.log2(txids.length) if (!Number.isInteger(sharedLevels)) throw new Error('benchmark proof group must be a power of two') const path = Array.from({ length: 24 }, (_, level) => { - const siblingHash = (BigInt(seed + 1) * 10_000n + BigInt(level + 1)) - .toString(16) - .padStart(64, '0') - .slice(-64) + const siblingHash = (BigInt(seed + 1) * 10_000n + BigInt(level + 1)).toString(16).padStart(64, '0').slice(-64) if (level === 0) return txids.map((hash, offset) => ({ offset, hash, txid: true })) if (level < sharedLevels) return [] return [{ offset: 1, hash: siblingHash }] @@ -82,27 +71,41 @@ function makeBenchmarkMerklePaths (txids: string[], height: number, seed: number return txids.map(txid => compound.extract([txid])) } -async function createBenchmarkContext (): Promise { +async function createBenchmarkContext(): Promise { const databaseName = process.env.WALLET_TOOLBOX_BENCH_MYSQL_DATABASE ?? 'createActionBeefBench' - const ctx = process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' - ? await _tu.createLegacyWalletMySQLCopy(databaseName, 'legacy') - : await _tu.createLegacyWalletSQLiteCopy(databaseName, 'legacy') + const ctx = + process.env.WALLET_TOOLBOX_BENCH_MYSQL === 'true' + ? await _tu.createLegacyWalletMySQLCopy(databaseName, 'legacy') + : await _tu.createLegacyWalletSQLiteCopy(databaseName, 'legacy') + const canonicalRoots = new Set() + // Synthetic benchmark proofs are valid only at their explicitly registered + // fixture heights. Never query a live chain or approve arbitrary roots. + jest.spyOn(ctx.activeStorage.getServices(), 'getChainTracker').mockResolvedValue({ + isValidRootForHeight: async (root, height) => canonicalRoots.has(`${height}:${root}`), + currentHeight: async () => 1_000_000 + }) const storageEvents: TelemetryEvent[] = [] - Reflect.set(ctx.activeStorage, 'telemetry', new Telemetry({ - sink: { capture: event => storageEvents.push({ ...event }) } - })) + Reflect.set( + ctx.activeStorage, + 'telemetry', + new Telemetry({ + sink: { capture: event => storageEvents.push({ ...event }) } + }) + ) ctx.activeStorage.feeModel = { model: 'sat/kb', value: 100 } - const basket = (await ctx.activeStorage.findOutputBaskets({ - partial: { userId: ctx.userId, name: 'default' } - }))[0] as TableOutputBasket + const basket = ( + await ctx.activeStorage.findOutputBaskets({ + partial: { userId: ctx.userId, name: 'default' } + }) + )[0] as TableOutputBasket await ctx.activeStorage.updateOutputBasket(basket.basketId, { numberOfDesiredUTXOs: 0, minimumDesiredUTXOValue: 1 }) - return { ctx, basket, storageEvents } + return { ctx, basket, storageEvents, canonicalRoots } } -function collectPhaseDurations (events: TelemetryEvent[], firstEvent: number): Record { +function collectPhaseDurations(events: TelemetryEvent[], firstEvent: number): Record { const phaseMs: Record = {} for (const event of events.slice(firstEvent)) { if (event.type !== 'span' || event.durationMs == null) continue @@ -113,7 +116,7 @@ function collectPhaseDurations (events: TelemetryEvent[], firstEvent: number): R return phaseMs } -async function replaceFundingCandidatesWithDistinctProvenSources ( +async function replaceFundingCandidatesWithDistinctProvenSources( setup: BenchmarkContext, candidateCount: number, outputSatoshis: number, @@ -129,10 +132,10 @@ async function replaceFundingCandidatesWithDistinctProvenSources ( const derivationSuffix = `beef-benchmark-${uniqueIndex}` const lockingScript = signableManagedOutputs ? new ScriptTemplateBRC29({ - derivationPrefix, - derivationSuffix, - keyDeriver: ctx.keyDeriver - }).lock(changeKeys.privateKey, changeKeys.publicKey) + derivationPrefix, + derivationSuffix, + keyDeriver: ctx.keyDeriver + }).lock(changeKeys.privateKey, changeKeys.publicKey) : Script.fromHex('51') const source = makeSourceTransaction(uniqueIndex, outputSatoshis, lockingScript) return { @@ -171,6 +174,7 @@ async function replaceFundingCandidatesWithDistinctProvenSources ( for (const preparedSource of prepared) { const { derivationSuffix, lockingScript, source, rawTx, txid } = preparedSource const merklePath = preparedSource.merklePath! + setup.canonicalRoots.add(`${merklePath.blockHeight}:${merklePath.computeRoot(txid)}`) const now = new Date() const proven: TableProvenTx = { created_at: now, @@ -228,24 +232,26 @@ async function replaceFundingCandidatesWithDistinctProvenSources ( return sourceTxids } -function createActionArgs (satoshis = 150_000) { +function createActionArgs(satoshis = 150_000) { return Validation.validateCreateActionArgs({ - outputs: [{ - satoshis, - lockingScript: '51', - outputDescription: 'proof-bearing benchmark output' - }], + outputs: [ + { + satoshis, + lockingScript: '51', + outputDescription: 'proof-bearing benchmark output' + } + ], description: 'createAction proof-bearing fragmented funding benchmark', options: { noSend: true, randomizeOutputs: false, returnTXIDOnly: false } }) } -function startQueryProbe (setup: BenchmarkContext): QueryProbe { +function startQueryProbe(setup: BenchmarkContext): QueryProbe { let queryCount = 0 let databaseTransactions = 0 let databaseMs = 0 - const started = new Map() - const countQuery = (query: { sql?: string, __knexQueryUid?: string }): void => { + const started = new Map() + const countQuery = (query: { sql?: string; __knexQueryUid?: string }): void => { queryCount++ if (/^begin\b/i.test(query.sql?.trim() ?? '')) databaseTransactions++ if (query.__knexQueryUid != null) { @@ -282,17 +288,11 @@ function startQueryProbe (setup: BenchmarkContext): QueryProbe { } } -async function measureStorageCreateAction ( - setup: BenchmarkContext, - candidateCount: number -): Promise { +async function measureStorageCreateAction(setup: BenchmarkContext, candidateCount: number): Promise { const firstEvent = setup.storageEvents.length const probe = startQueryProbe(setup) const start = performance.now() - const result = await setup.ctx.activeStorage.createAction( - { userId: setup.ctx.userId }, - createActionArgs() - ) + const result = await setup.ctx.activeStorage.createAction({ userId: setup.ctx.userId }, createActionArgs()) const elapsedMs = performance.now() - start const query = probe.stop() return { @@ -306,9 +306,7 @@ async function measureStorageCreateAction ( } } -async function createRemoteClient ( - setup: BenchmarkContext -): Promise<{ +async function createRemoteClient(setup: BenchmarkContext): Promise<{ client: TestWalletOnly server: StorageServer verifier: BdkVerifier @@ -356,7 +354,7 @@ async function createRemoteClient ( return { client, server, verifier, verifyDigestBatch, events, serverEvents } } -async function measureRemoteWalletCreateAction ( +async function measureRemoteWalletCreateAction( setup: BenchmarkContext, client: TestWalletOnly, events: TelemetryEvent[], @@ -369,11 +367,13 @@ async function measureRemoteWalletCreateAction ( const probe = startQueryProbe(setup) const start = performance.now() const result = await client.wallet.createAction({ - outputs: [{ - satoshis: 150_000, - lockingScript: '51', - outputDescription: 'proof-bearing benchmark output' - }], + outputs: [ + { + satoshis: 150_000, + lockingScript: '51', + outputDescription: 'proof-bearing benchmark output' + } + ], description: 'authenticated remote proof-bearing fragmented funding benchmark', options: { noSend: true, randomizeOutputs: false, returnTXIDOnly: false } }) @@ -403,12 +403,12 @@ async function measureRemoteWalletCreateAction ( } } -function percentile (values: number[], percentileValue: number): number { +function percentile(values: number[], percentileValue: number): number { const sorted = [...values].sort((a, b) => a - b) return sorted[Math.max(0, Math.ceil(percentileValue * sorted.length) - 1)] } -function summarize (measurements: Measurement[]): MeasurementSummary { +function summarize(measurements: Measurement[]): MeasurementSummary { const elapsed = measurements.map(measurement => measurement.elapsedMs) const database = measurements.map(measurement => measurement.databaseMs) const phaseNames = new Set(measurements.flatMap(measurement => Object.keys(measurement.phaseMs ?? {}))) @@ -473,13 +473,12 @@ describe('createAction proof-bearing fragmented funding benchmark', () => { readEnabled: true, writeEnabled: true }) - jest.spyOn(setup.ctx.activeStorage.getServices(), 'getChainTracker').mockResolvedValue({ - isValidRootForHeight: async () => true - }) - expect(setup.ctx.activeStorage.enqueuePreparedBeef({ - userId: setup.ctx.userId, - rootTxids: warmTxids - })).toBe(true) + expect( + setup.ctx.activeStorage.enqueuePreparedBeef({ + userId: setup.ctx.userId, + rootTxids: warmTxids + }) + ).toBe(true) await setup.ctx.activeStorage.waitForPreparedBeefTasks() const canonicalBuilder = jest.spyOn(setup.ctx.activeStorage, 'getBeefForTransactions') const prepared = await measureStorageCreateAction(setup, warmTxids.length) @@ -512,42 +511,40 @@ describe('createAction proof-bearing fragmented funding benchmark', () => { for (let sample = 0; sample < samples; sample++) { await replaceFundingCandidatesWithDistinctProvenSources(setup, 178, 1_000, sample + 1_000, true) await new Promise(resolve => setTimeout(resolve, 1_250)) - remoteMeasurements.push(await measureRemoteWalletCreateAction( - setup, - remote.client, - remote.events, - remote.serverEvents, - 178 - )) + remoteMeasurements.push( + await measureRemoteWalletCreateAction(setup, remote.client, remote.events, remote.serverEvents, 178) + ) } const typicalRemoteMeasurements: Measurement[] = [] for (let sample = 0; sample < samples; sample++) { await replaceFundingCandidatesWithDistinctProvenSources(setup, 8, 200_000, sample + 2_000, true) await new Promise(resolve => setTimeout(resolve, 1_250)) - typicalRemoteMeasurements.push(await measureRemoteWalletCreateAction( - setup, - remote.client, - remote.events, - remote.serverEvents, - 8 - )) + typicalRemoteMeasurements.push( + await measureRemoteWalletCreateAction(setup, remote.client, remote.events, remote.serverEvents, 8) + ) } const direct = summarize(directMeasurements) const authenticatedRemote = summarize(remoteMeasurements) const typicalAuthenticatedRemote = summarize(typicalRemoteMeasurements) - const digestVerdicts = (await Promise.all( - remote.verifyDigestBatch.mock.results.map(async result => await result.value) - )).flat() - process.stdout.write(`${JSON.stringify({ - direct, - authenticatedRemote, - typicalAuthenticatedRemote, - digestVerificationBatches: remote.verifyDigestBatch.mock.calls.length, - digestVerificationCount: digestVerdicts.length, - digestVerificationFailures: digestVerdicts.filter(valid => !valid).length - }, null, 2)}\n`) + const digestVerdicts = ( + await Promise.all(remote.verifyDigestBatch.mock.results.map(async result => await result.value)) + ).flat() + process.stdout.write( + `${JSON.stringify( + { + direct, + authenticatedRemote, + typicalAuthenticatedRemote, + digestVerificationBatches: remote.verifyDigestBatch.mock.calls.length, + digestVerificationCount: digestVerdicts.length, + digestVerificationFailures: digestVerdicts.filter(valid => !valid).length + }, + null, + 2 + )}\n` + ) expect(direct.p50Ms).toBeLessThan(150) expect(direct.p95Ms).toBeLessThan(500) expect(authenticatedRemote.p50Ms).toBeLessThan(450) @@ -617,17 +614,22 @@ describe('createAction proof-bearing fragmented funding benchmark', () => { resultBeefBytes: result.inputBeef?.length ?? 0, phaseMs }) - await expect(storage.abortAction( - { userId: user.userId }, - { reference: result.reference } - )).resolves.toEqual({ aborted: true }) + await expect(storage.abortAction({ userId: user.userId }, { reference: result.reference })).resolves.toEqual({ + aborted: true + }) } const summary = summarize(measurements) - process.stdout.write(`${JSON.stringify({ - productionShaped: summary, - minSelectedInputCount: Math.min(...measurements.map(measurement => measurement.selectedInputCount)), - minDistinctSourceCount: Math.min(...measurements.map(measurement => measurement.distinctSourceCount)) - }, null, 2)}\n`) + process.stdout.write( + `${JSON.stringify( + { + productionShaped: summary, + minSelectedInputCount: Math.min(...measurements.map(measurement => measurement.selectedInputCount)), + minDistinctSourceCount: Math.min(...measurements.map(measurement => measurement.distinctSourceCount)) + }, + null, + 2 + )}\n` + ) expect(measurements.every(measurement => measurement.candidateCount > 100)).toBe(true) expect(measurements.every(measurement => measurement.selectedInputCount > 100)).toBe(true) expect(measurements.every(measurement => measurement.resultBeefBytes > 0)).toBe(true) diff --git a/packages/wallet/wallet-toolbox/benchmarks/resumable-sync.bench.test.ts b/packages/wallet/wallet-toolbox/benchmarks/resumable-sync.bench.test.ts new file mode 100644 index 000000000..3bc9e2e7d --- /dev/null +++ b/packages/wallet/wallet-toolbox/benchmarks/resumable-sync.bench.test.ts @@ -0,0 +1,248 @@ +import 'fake-indexeddb/auto' +import { once } from 'node:events' +import { randomUUID } from 'node:crypto' +import { performance } from 'node:perf_hooks' +import { PrivateKey, ProtoWallet } from '@bsv/sdk' +import { StorageIdb } from '../src/storage/StorageIdb' +import { StorageKnex } from '../src/storage/StorageKnex' +import { StorageProvider } from '../src/storage/StorageProvider' +import { WalletStorageManager } from '../src/storage/WalletStorageManager' +import { StorageClient } from '../src/storage/remoting/StorageClient' +import { StorageServer } from '../src/storage/remoting/StorageServer' +import { KnexSessionManager } from '../src/storage/remoting/KnexSessionManager' +import { _tu } from '../test/utils/TestUtilsWalletStorage' +import type { SyncSessionProgress } from '../src/storage/sync/syncSession' +import type { WalletStorageSyncReader } from '../src/sdk/WalletStorage.interfaces' + +import { labels, proofCount, proofBytes, pageBytes, fixedReadMs, seedSyncBenchmark } from './sync-fixture' + +const delay = async (ms: number) => await new Promise(resolve => setTimeout(resolve, ms)) +const percentile = (values: number[], fraction: number) => + [...values].sort((a, b) => a - b)[Math.max(0, Math.ceil(values.length * fraction) - 1)] ?? 0 + +async function store(kind: 'idb' | 'sqlite'): Promise { + const storage = + kind === 'idb' + ? new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + : new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: _tu.createLocalSQLite(':memory:') + }) + if (storage instanceof StorageIdb) storage.dbName = `sync-benchmark-${randomUUID()}` + await storage.migrate('bounded sync benchmark', PrivateKey.fromRandom().toPublicKey().toString()) + await storage.makeAvailable() + return storage +} +async function close(storage: StorageProvider) { + await storage.destroy() + if (storage instanceof StorageIdb) await storage.dropAllData() +} + +describe.each(['sqlite', 'http'] as const)('bounded sync measured on %s', backend => { + test('copies large proofs and same-timestamp tombstones while foreground work progresses', async () => { + const source = await store('sqlite') + const key = PrivateKey.fromRandom() + const identityKey = key.toPublicKey().toString() + process.stdout.write(`seeding ${backend}\n`) + const user = await seedSyncBenchmark(source, identityKey) + process.stdout.write(`seeded ${backend}\n`) + let reader: WalletStorageSyncReader = source + let server: StorageServer | undefined + let wireBytes = 0 + let queries = 0 + if (source instanceof StorageKnex) + source.knex.on('query', () => { + queries++ + }) + if (backend === 'http') { + server = new StorageServer(source, { + port: 0, + wallet: new ProtoWallet(key), + monetize: false, + logRpcRequests: false, + sessionManager: new KnexSessionManager((source as StorageKnex).knex), + adminIdentityKeys: [], + calculateRequestPrice: async () => 0 + }) + server.start() + if (!server.server.listening) await once(server.server, 'listening') + server.server.on('request', (_req, res) => { + const write = res.write.bind(res) + const end = res.end.bind(res) + const size = (chunk: unknown) => + typeof chunk === 'string' ? Buffer.byteLength(chunk) : chunk instanceof Uint8Array ? chunk.byteLength : 0 + res.write = ((chunk: unknown, ...args: unknown[]) => { + wireBytes += size(chunk) + return Reflect.apply(write, res, [chunk, ...args]) + }) as typeof res.write + res.end = ((chunk: unknown, ...args: unknown[]) => { + wireBytes += size(chunk) + return Reflect.apply(end, res, [chunk, ...args]) + }) as typeof res.end + }) + const address = server.server.address() + if (address == null || typeof address === 'string') throw new Error('HTTP fixture did not bind') + reader = new StorageClient(new ProtoWallet(key), `http://127.0.0.1:${address.port}`) + } + const getChunk = reader.getSyncChunk.bind(reader) + let pages = 0 + let largestPage = 0 + reader.getSyncChunk = async args => { + expect(args.maxItems).toBeLessThanOrEqual(1000) + expect(args.maxRoughSize).toBeLessThanOrEqual(pageBytes) + await delay(fixedReadMs) + const chunk = await getChunk(args) + pages++ + largestPage = Math.max( + largestPage, + Object.values(chunk).reduce((sum, value) => sum + (Array.isArray(value) ? value.length : 0), 0) + ) + return chunk + } + const reports: Record[] = [] + try { + for (const mode of ['exclusive', 'paged'] as const) { + const destination = await store('sqlite') + if (destination instanceof StorageKnex) + destination.knex.on('query', () => { + queries++ + }) + const capabilities = destination.getCapabilities.bind(destination) + if (mode === 'exclusive') + destination.getCapabilities = async () => ({ ...(await capabilities()), storageAccess: undefined }) + const manager = new WalletStorageManager(identityKey, destination) + await manager.makeAvailable() + const userId = await manager.getUserId() + let active = true + let pending = false + const foreground: number[] = [] + const eventLoopDelay: number[] = [] + let expectedTick = performance.now() + 5 + const progress: SyncSessionProgress[] = [] + let peakRss = process.memoryUsage().rss + const startRss = peakRss + const startCpu = process.cpuUsage() + const beforePages = pages + const beforeQueries = queries + const beforeWire = wireBytes + let foregroundTask: Promise = Promise.resolve() + const timer = setInterval(() => { + const now = performance.now() + eventLoopDelay.push(Math.max(0, now - expectedTick)) + expectedTick = now + 5 + peakRss = Math.max(peakRss, process.memoryUsage().rss) + if (!active || pending) return + pending = true + const started = performance.now() + foregroundTask = manager + .runAsReader( + async () => await destination.findTxLabels({ partial: { userId }, paged: { limit: 20, offset: 0 } }) + ) + .then(async () => { + await manager.runAsWriter(async () => await destination.findOrInsertOutputBasket(userId, 'foreground')) + }) + .then(() => { + foreground.push(performance.now() - started) + }) + .finally(() => { + pending = false + }) + }, 5) + process.stdout.write(`copying ${backend} ${mode}\n`) + const start = performance.now() + try { + const full = await manager.syncFromReaderResumable(identityKey, reader, { + maxRoughSize: pageBytes, + onProgress: event => { + progress.push(event) + if (event.state === 'committed') + process.stdout.write( + `page ${backend} ${mode} ${event.pages}: ${event.inserts} inserted, ${event.commitMs} ms commit\n` + ) + } + }) + const wallMs = performance.now() - start + active = false + clearInterval(timer) + await foregroundTask + const cpu = process.cpuUsage(startCpu) + expect(full).toMatchObject({ status: 'completed', mode }) + expect(await destination.countTxLabels({ partial: { userId } })).toBe(labels) + expect(await destination.countTxLabels({ partial: { userId, isDeleted: true } })).toBe(Math.ceil(labels / 97)) + expect(await destination.countProvenTxs({ partial: {} })).toBe(proofCount) + const report = { + backend, + mode, + labels, + proofCount, + proofBytes, + pageBytes, + fixedReadMs, + wallMs, + cpuMs: (cpu.user + cpu.system) / 1000, + startRss, + peakRss, + rssGrowth: peakRss - startRss, + pages: pages - beforePages, + largestPage, + sqlQueries: queries - beforeQueries, + responseBodyBytes: backend === 'http' ? wireBytes - beforeWire : null, + foreground: { + samples: foreground.length, + p50Ms: percentile(foreground, 0.5), + p95Ms: percentile(foreground, 0.95), + p99Ms: percentile(foreground, 0.99) + }, + eventLoopDelay: { + samples: eventLoopDelay.length, + p95Ms: percentile(eventLoopDelay, 0.95), + p99Ms: percentile(eventLoopDelay, 0.99), + maxMs: Math.max(0, ...eventLoopDelay) + }, + maxQueueMs: Math.max(0, ...progress.map(event => event.queueMs ?? 0)), + maxCommitMs: Math.max(0, ...progress.map(event => event.commitMs ?? 0)) + } + reports.push(report) + if (mode === 'paged') { + expect(foreground.length).toBeGreaterThan(8) + expect(percentile(eventLoopDelay, 0.95)).toBeLessThan(1500) + expect(percentile(foreground, 0.95)).toBeLessThan(1500) + } + expect(wallMs).toBeLessThan(120000) + expect(peakRss - startRss).toBeLessThan(512 * 1024 * 1024) + const quiet = await manager.syncFromReaderResumable(identityKey, reader, { maxRoughSize: pageBytes }) + expect(quiet).toMatchObject({ inserts: 0, updates: 0 }) + // Inclusive legacy timestamps deliberately reread the final boundary. + // An entire imported batch can share that timestamp; preserve late peers. + expect(quiet.pages).toBeLessThanOrEqual(full.pages + 1) + Object.assign(report, { unchangedBoundaryPages: quiet.pages }) + const [changed] = await source.findTxLabels({ partial: { userId: user.userId, label: 'label 1' } }) + await source.updateTxLabel(changed.txLabelId, { isDeleted: !changed.isDeleted, updated_at: new Date() }) + const incremental = await manager.syncFromReaderResumable(identityKey, reader, { maxRoughSize: pageBytes }) + expect(incremental.updates).toBe(1) + expect(incremental.pages).toBeLessThanOrEqual(full.pages + 1) + const settled = await manager.syncFromReaderResumable(identityKey, reader, { maxRoughSize: pageBytes }) + expect(settled).toMatchObject({ inserts: 0, updates: 0 }) + expect(settled.pages).toBeLessThanOrEqual(2) + Object.assign(report, { incrementalPages: incremental.pages, settledUnchangedPages: settled.pages }) + await source.updateTxLabel(changed.txLabelId, { + isDeleted: changed.isDeleted, + updated_at: changed.updated_at + }) + } finally { + active = false + clearInterval(timer) + await foregroundTask + await close(destination) + } + } + const [exclusive, paged] = reports as Array<{ wallMs: number; foreground: { p95Ms: number } }> + expect(paged.foreground.p95Ms).toBeLessThan(exclusive.foreground.p95Ms / 2) + expect(paged.wallMs).toBeLessThan(exclusive.wallMs * 2) + } finally { + process.stdout.write(`${JSON.stringify({ resumableSync: reports }, null, 2)}\n`) + if (server !== undefined) await server.close() + await close(source) + } + }, 300000) +}) diff --git a/packages/wallet/wallet-toolbox/benchmarks/sync-fixture.ts b/packages/wallet/wallet-toolbox/benchmarks/sync-fixture.ts new file mode 100644 index 000000000..6bc6ee757 --- /dev/null +++ b/packages/wallet/wallet-toolbox/benchmarks/sync-fixture.ts @@ -0,0 +1,78 @@ +import { runInSeries } from '../src/utility/runInSeries' +import { Script, Transaction, MerklePath } from '@bsv/sdk' +import type { StorageProvider } from '../src/storage/StorageProvider' + +export const labels = 10000 +export const proofCount = 64 +export const proofBytes = 32 * 1024 +export const pageBytes = 256 * 1024 +export const fixedReadMs = 15 + +function* fixtureIndexes(count: number) { + for (let index = 0; index < count; index++) yield index +} + +export async function seedSyncBenchmark( + storage: Pick< + StorageProvider, + 'findOrInsertUser' | 'transaction' | 'insertTxLabel' | 'insertProvenTx' | 'insertTransaction' + >, + identityKey: string +) { + const { user } = await storage.findOrInsertUser(identityKey) + const timestamp = new Date(1_700_000_000_000) + await storage.transaction(async trx => { + // Seed in order on one transaction without queuing thousands of pending writes. + await runInSeries(fixtureIndexes(labels), i => + storage.insertTxLabel( + { + txLabelId: 0, + userId: user.userId, + label: `label ${i}`, + isDeleted: i % 97 === 0, + created_at: timestamp, + updated_at: timestamp + }, + trx + ) + ) + await runInSeries(fixtureIndexes(proofCount), async i => { + const tx = new Transaction() + tx.addOutput({ satoshis: i + 1, lockingScript: Script.fromASM(`OP_FALSE OP_RETURN ${'01'.repeat(proofBytes)}`) }) + const txid = tx.id('hex') + const path = new MerklePath(100, [[{ offset: 0, hash: txid, txid: true }]]) + const provenTxId = await storage.insertProvenTx( + { + provenTxId: 0, + txid, + rawTx: tx.toBinary(), + merklePath: path.toBinary(), + merkleRoot: txid, + height: 100, + index: 0, + blockHash: '01'.repeat(32), + created_at: timestamp, + updated_at: timestamp + }, + trx + ) + await storage.insertTransaction( + { + transactionId: 0, + userId: user.userId, + provenTxId, + txid, + reference: `bounded-sync-${i}`, + status: 'completed', + isOutgoing: false, + satoshis: i + 1, + description: 'large proof sync fixture', + created_at: timestamp, + updated_at: timestamp + }, + trx + ) + }) + }) + return user +} diff --git a/packages/wallet/wallet-toolbox/client/README.md b/packages/wallet/wallet-toolbox/client/README.md index f2d7ef5aa..47bd1fe19 100644 --- a/packages/wallet/wallet-toolbox/client/README.md +++ b/packages/wallet/wallet-toolbox/client/README.md @@ -13,6 +13,20 @@ Use this package in: For Node servers, use [`@bsv/wallet-toolbox`](https://www.npmjs.com/package/@bsv/wallet-toolbox). For React Native / mobile, use [`@bsv/wallet-toolbox-mobile`](https://www.npmjs.com/package/@bsv/wallet-toolbox-mobile). +## Resumable synchronization (2.15 candidate) + +`WalletStorageManager.syncFromReaderResumable(identityKey, source, options)` adds +cancellation, durable checkpoints and per-page progress. IndexedDB and Knex local +destinations yield the manager queue during source/proof I/O and between atomic +page commits. Remote destinations retain exclusive execution. Use +`maxRoughSize: 262144` as a measured starting point for constrained clients; +one oversized record may still exceed this rough page target and is subject to +the provider's separate transfer bound. Resume by invoking the API again: the +destination checkpoint is authoritative, including after a lost acknowledgement. +See the [sync contract and next-stage design](../../../../docs/guides/wallet-sync-reliability.md). +This is an eventual replica merge; it does not create a coherent source snapshot +or change the existing archive format. + ## BRC-100 result compatibility Version 2.14.1 keeps internal exact-spend accounting off public `createAction` @@ -32,6 +46,13 @@ data copy and test recovery on a clean profile. Read [Wallet backup and recovery](https://bsv-blockchain.github.io/ts-stack/guides/wallet-backup-recovery/), [BRC-38/39 integration](https://bsv-blockchain.github.io/ts-stack/guides/wallet-data-portability/) and the [recovery checklist](https://bsv-blockchain.github.io/ts-stack/guides/wallet-recovery-drill/). +The unpublished candidate exports `RetainedReadSnapshot` and +`RetainedReadSnapshotOptions` types. They do not add a retained IndexedDB, +remote, or native mobile implementation. Unsupported local providers report +`supportsRetainedReadSnapshot() === false` and refuse `openReadSnapshot()`. +The SQL implementation and its connection/lifetime limits are described in the +[sync guide](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#retained-local-sql-read-views-unpublished-candidate). + Portable helpers require a concrete local `StorageProvider`; a remote client is not one. Qualify the local-copy path and device memory limits before adding export/import UI. @@ -206,3 +227,27 @@ reference, reclaim, commission, and relation lookups avoid repeated full-wallet scans during restores. Legacy duplicate transaction IDs remain intact. Clients that request an older IndexedDB schema version cannot reopen this database; retain a compatible client when using the local backup. + +The 2.15 candidate also exports version-one `WalletReadSnapshot`, cursor/page and +packed-row types. The base provider explicitly refuses unsupported local pages; +only `StorageKnex` currently implements them. Type availability does not imply an +IndexedDB, native mobile or remote snapshot implementation. Existing sync and +archive APIs retain their behavior. See the [SQL paging contract](../../../../docs/guides/wallet-sync-reliability.md#profile-bound-local-sql-pages). + +The candidate also exports `snapshotSyncTables` and the version-one +`SnapshotSyncSource`, `SnapshotSyncCheckpoint`, `SnapshotSyncCommit`, +`SnapshotSyncStorage` and `SnapshotSyncTable` types. `syncToWriterResumable` adds +push progress/cancellation with an explicit exclusive fallback. These exports +support adapter integration; they do not enable SQL retention or the auxiliary +SQL migration in this browser/mobile entry point. Current IndexedDB and remote +paths retain their documented behavior. See the [local SQL integration and remaining limits](../../../../docs/guides/wallet-sync-reliability.md#durable-local-sql-sync-and-ordinary-backup). + +The candidate now includes `getSnapshotArchiveTransport(identityKey)` on +`StorageClient`. It negotiates a migrated WAL/MySQL server's immutable archive +capability and validates bounded authenticated admission/status, directories and +pages. A dedicated response ceiling applies before parsing; server and storage +identity bindings remain shared with ordinary RPC. `snapshotArchives: false` +disables the transport, and old/disabled peers decline it. This is a low-level +transport, not a retained database adapter or adoption by ordinary sync/export. +Remote row cursors, manager integration, portable semantics and physical platform +acceptance remain open. See the [archive transport contract](../../../../docs/guides/wallet-sync-reliability.md#authenticated-snapshot-archive-transport-unpublished-candidate). diff --git a/packages/wallet/wallet-toolbox/client/package.json b/packages/wallet/wallet-toolbox/client/package.json index 89e4036fe..f4b11dcf6 100644 --- a/packages/wallet/wallet-toolbox/client/package.json +++ b/packages/wallet/wallet-toolbox/client/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-client", - "version": "2.14.5", + "version": "2.15.0", "type": "module", "sideEffects": false, "engines": { @@ -56,11 +56,12 @@ "lint": "pnpm --workspace-root exec oxlint packages/wallet/wallet-toolbox/src/index.client.ts packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/index.client.ts packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/index.mobile.ts packages/wallet/wallet-toolbox/src/services/chaintracker/chaintracks/Api/BlockHeaderApi.ts packages/wallet/wallet-toolbox/client/test packages/wallet/wallet-toolbox/client/tsdown.config.ts packages/wallet/wallet-toolbox/client/vitest.config.ts --deny-warnings", "pack:check": "node ../../../../scripts/check-package-artifact.mjs . --exports Wallet,WalletSigner,WalletStorageManager,StorageClient,StorageIdb,Services,SetupClient,WalletPermissionsManager,WalletSettingsManager,LocalChainTracker,FixedWindowBulkFileDownloadBudget,InlineBulkFileDataValidator,registerArgon2idBackend,unregisterArgon2idBackend,WalletMonitorTask,attemptToPostReqsToNetwork,parseJsonRpc,stringifyJsonRpc,verifyUnlockScripts,sdk", "test": "vitest run", - "test:browser": "node ../../../../scripts/check-wallet-toolbox-platform.mjs browser", + "test:browser": "node ../../../../scripts/check-wallet-toolbox-platform.mjs browser && node test/sync-browser.mjs", "test:coverage": "vitest run --coverage", "test:watch": "vitest", "typecheck": "tsc --project tsconfig.typecheck.json", - "prepublishOnly": "npm run build" + "prepublishOnly": "npm run build", + "bench:sync": "pnpm build && node test/sync-browser.mjs" }, "bugs": { "url": "https://github.com/bsv-blockchain/ts-stack/issues" @@ -89,6 +90,7 @@ "tsdown": "0.23.0", "typescript": "npm:@typescript/typescript6@6.0.2", "vite": "8.3.0", - "vitest": "^5.0.1" + "vitest": "^5.0.1", + "puppeteer-core": "^25.4.0" } } diff --git a/packages/wallet/wallet-toolbox/client/test/sync-browser.mjs b/packages/wallet/wallet-toolbox/client/test/sync-browser.mjs new file mode 100644 index 000000000..d27f16e32 --- /dev/null +++ b/packages/wallet/wallet-toolbox/client/test/sync-browser.mjs @@ -0,0 +1,60 @@ +import assert from 'node:assert/strict' +import { createServer } from 'node:http' +import { access } from 'node:fs/promises' +import { fileURLToPath } from 'node:url' +import { build } from 'esbuild' +import puppeteer from 'puppeteer-core' + +const candidates = [ + process.env.CHROME_BIN, + '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome', + '/usr/bin/google-chrome', + '/usr/bin/chromium' +].filter(Boolean) +let executablePath +for (const candidate of candidates) { + try { + await access(candidate) + executablePath = candidate + break + } catch { + /* try the next installed browser */ + } +} +assert.ok(executablePath, 'Install Chromium/Chrome or set CHROME_BIN; the native IndexedDB gate cannot be skipped.') +const bundle = await build({ + entryPoints: [fileURLToPath(new URL('./sync-browser.ts', import.meta.url))], + bundle: true, + write: false, + platform: 'browser', + format: 'esm', + target: 'es2022' +}) +const server = createServer((req, res) => { + if (req.url === '/benchmark.js') { + res.setHeader('Content-Type', 'text/javascript') + res.end(bundle.outputFiles[0].contents) + } else { + res.setHeader('Content-Type', 'text/html') + res.end('Native sync acceptance') + } +}) +await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)) +let browser +try { + browser = await puppeteer.launch({ + executablePath, + headless: true, + args: ['--no-sandbox', '--enable-precise-memory-info'] + }) + const page = await browser.newPage() + page.on('console', message => process.stdout.write(`${message.text()}\n`)) + page.on('pageerror', error => process.stderr.write(`${error}\n`)) + await page.goto(`http://127.0.0.1:${server.address().port}`) + await page.waitForFunction('typeof globalThis.syncBenchmark === "function"', { timeout: 30000 }) + const reports = await page.evaluate(async () => await globalThis.syncBenchmark()) + process.stdout.write(`${JSON.stringify({ nativeSync: reports }, null, 2)}\n`) +} finally { + if (browser !== undefined) await browser.close() + await new Promise((resolve, reject) => server.close(error => (error ? reject(error) : resolve()))) +} diff --git a/packages/wallet/wallet-toolbox/client/test/sync-browser.ts b/packages/wallet/wallet-toolbox/client/test/sync-browser.ts new file mode 100644 index 000000000..b03b5af52 --- /dev/null +++ b/packages/wallet/wallet-toolbox/client/test/sync-browser.ts @@ -0,0 +1,193 @@ +import { PrivateKey } from '@bsv/sdk' +import { StorageIdb, StorageProvider, WalletStorageManager, type SyncSessionProgress } from '@bsv/wallet-toolbox-client' +import { + fixedReadMs, + labels, + pageBytes, + proofBytes, + proofCount, + seedSyncBenchmark +} from '../../benchmarks/sync-fixture' + +const percentile = (values: number[], fraction: number) => + [...values].sort((a, b) => a - b)[Math.max(0, Math.ceil(values.length * fraction) - 1)] ?? 0 +const check = (condition: boolean, message: string) => { + if (!condition) throw new Error(message) +} +async function open(): Promise { + const storage = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + storage.dbName = `native-sync-${crypto.randomUUID()}` + await storage.migrate('native browser sync benchmark', PrivateKey.fromRandom().toPublicKey().toString()) + await storage.makeAvailable() + return storage +} +async function close(storage: StorageIdb) { + await storage.destroy() + await storage.dropAllData() +} + +async function* destinationModes() { + for (const mode of ['exclusive', 'paged'] as const) { + yield open().then(destination => ({ mode, destination })) + } +} + +async function benchmark() { + const source = await open() + const identityKey = PrivateKey.fromRandom().toPublicKey().toString() + const user = await seedSyncBenchmark(source, identityKey) + const reports = [] + let reads = 0 + let maxRows = 0 + const read = source.getSyncChunk.bind(source) + source.getSyncChunk = async args => { + check(args.maxItems <= 1000 && args.maxRoughSize <= pageBytes, 'page exceeded configured bounds') + await new Promise(resolve => setTimeout(resolve, fixedReadMs)) + const chunk = await read(args) + reads++ + maxRows = Math.max( + maxRows, + Object.values(chunk).reduce((sum, value) => sum + (Array.isArray(value) ? value.length : 0), 0) + ) + return chunk + } + try { + // Modes share a source fixture and must be measured without competing workloads. + for await (const { mode, destination } of destinationModes()) { + const capabilities = destination.getCapabilities.bind(destination) + if (mode === 'exclusive') + destination.getCapabilities = async () => ({ ...(await capabilities()), storageAccess: undefined }) + const manager = new WalletStorageManager(identityKey, destination) + await manager.makeAvailable() + const userId = await manager.getUserId() + let active = true + let pending = false + let foregroundTask = Promise.resolve() + const foreground: number[] = [] + const eventLoopDelay: number[] = [] + let expectedTick = performance.now() + 5 + const progress: SyncSessionProgress[] = [] + const startReads = reads + const heap = () => + (performance as Performance & { memory?: { usedJSHeapSize: number } }).memory?.usedJSHeapSize ?? 0 + const startHeap = heap() + let peakHeap = startHeap + const timer = setInterval(() => { + const now = performance.now() + eventLoopDelay.push(Math.max(0, now - expectedTick)) + expectedTick = now + 5 + peakHeap = Math.max(peakHeap, heap()) + if (!active || pending) return + pending = true + const started = performance.now() + foregroundTask = manager + .runAsReader( + async () => await destination.findTxLabels({ partial: { userId }, paged: { offset: 0, limit: 20 } }) + ) + .then(async () => { + await manager.runAsWriter(async () => await destination.findOrInsertOutputBasket(userId, 'foreground')) + }) + .then(() => { + foreground.push(performance.now() - started) + }) + .finally(() => { + pending = false + }) + }, 5) + const start = performance.now() + try { + const copied = await manager.syncFromReaderResumable(identityKey, source, { + maxRoughSize: pageBytes, + onProgress: event => progress.push(event) + }) + const wallMs = performance.now() - start + active = false + clearInterval(timer) + await foregroundTask + check(copied.status === 'completed' && copied.mode === mode, 'copy did not complete in expected mode') + check((await destination.countTxLabels({ partial: { userId } })) === labels, 'label count differs') + check( + (await destination.countTxLabels({ partial: { userId, isDeleted: true } })) === Math.ceil(labels / 97), + 'same-timestamp tombstones were lost' + ) + check((await destination.countProvenTxs({ partial: {} })) === proofCount, 'large proofs were lost') + const report = { + backend: 'Chromium IndexedDB', + mode, + labels, + proofCount, + proofBytes, + pageBytes, + fixedReadMs, + wallMs, + pages: reads - startReads, + maxRows, + startHeap, + peakHeap, + foreground: { + samples: foreground.length, + p50Ms: percentile(foreground, 0.5), + p95Ms: percentile(foreground, 0.95), + p99Ms: percentile(foreground, 0.99) + }, + eventLoopDelay: { + samples: eventLoopDelay.length, + p95Ms: percentile(eventLoopDelay, 0.95), + p99Ms: percentile(eventLoopDelay, 0.99), + maxMs: Math.max(0, ...eventLoopDelay) + }, + maxQueueMs: Math.max(0, ...progress.map(event => event.queueMs ?? 0)), + maxCommitMs: Math.max(0, ...progress.map(event => event.commitMs ?? 0)), + maxReadMs: Math.max(0, ...progress.map(event => event.readMs ?? 0)) + } + reports.push(report) + console.log(JSON.stringify(report)) + check(wallMs < 120000, 'full copy exceeded 120 seconds') + check(peakHeap - startHeap < 512 * 1024 * 1024, 'heap growth exceeded 512 MiB') + if (mode === 'paged') { + check(foreground.length > 8, 'background sync starved foreground work') + check(percentile(eventLoopDelay, 0.95) < 1500, 'event loop p95 exceeded 1.5 seconds') + check(percentile(foreground, 0.95) < 1500, 'foreground p95 exceeded 1.5 seconds') + } + const quiet = await manager.syncFromReaderResumable(identityKey, source, { maxRoughSize: pageBytes }) + check( + quiet.inserts === 0 && quiet.updates === 0 && quiet.pages <= copied.pages + 1, + 'unchanged boundary replay mutated rows or exceeded a full pass' + ) + const [changed] = await source.findTxLabels({ partial: { userId: user.userId, label: 'label 1' } }) + await source.updateTxLabel(changed.txLabelId, { isDeleted: true, updated_at: new Date() }) + const incremental = await manager.syncFromReaderResumable(identityKey, source, { maxRoughSize: pageBytes }) + check( + incremental.updates === 1 && incremental.pages <= copied.pages + 1, + 'incremental tombstone did not propagate' + ) + const settled = await manager.syncFromReaderResumable(identityKey, source, { maxRoughSize: pageBytes }) + check( + settled.inserts === 0 && settled.updates === 0 && settled.pages <= 2, + 'settled unchanged copy was not quiet' + ) + Object.assign(report, { + unchangedBoundaryPages: quiet.pages, + incrementalPages: incremental.pages, + settledUnchangedPages: settled.pages + }) + await source.updateTxLabel(changed.txLabelId, { isDeleted: false, updated_at: changed.updated_at }) + } finally { + active = false + clearInterval(timer) + await foregroundTask + await close(destination) + } + } + check( + reports[1].foreground.p95Ms < reports[0].foreground.p95Ms / 2, + 'foreground p95 did not improve by at least 2x' + ) + check(reports[1].wallMs < reports[0].wallMs * 2, 'paged copy doubled total time') + return reports + } finally { + await close(source) + } +} + +Object.assign(globalThis, { syncBenchmark: benchmark }) diff --git a/packages/wallet/wallet-toolbox/jest.config.cjs b/packages/wallet/wallet-toolbox/jest.config.cjs index 04aa4c774..bd344f516 100644 --- a/packages/wallet/wallet-toolbox/jest.config.cjs +++ b/packages/wallet/wallet-toolbox/jest.config.cjs @@ -13,7 +13,12 @@ const getJestConfig = async () => { // Speed up by restricting to module (source files) extensions used. moduleFileExtensions: ['ts', 'js'], // excluded source files... - modulePathIgnorePatterns: ['out/src', 'out/test', '/dist/cjs/'], + modulePathIgnorePatterns: ['out/src', 'out/test', '/dist/cjs/', String.raw`/\.stryker-tmp/`], + // Ignore generated children of this test root. Stryker runs with a sandbox + // as its root, so an unanchored sandbox pattern would hide its own tests. + // Module discovery retains the same boundary when package commands replace + // testPathIgnorePatterns with their governed manual/live test policy. + testPathIgnorePatterns: ['/node_modules/', String.raw`/\.stryker-tmp/`], // Default is 'node' testEnvironment: 'node', // default [ '**/__tests__/**/*.[jt]s?(x)', '**/?(*.)+(spec|test).[tj]s?(x)' ] diff --git a/packages/wallet/wallet-toolbox/mobile/README.md b/packages/wallet/wallet-toolbox/mobile/README.md index 84d5d7c4b..1ab5744ac 100644 --- a/packages/wallet/wallet-toolbox/mobile/README.md +++ b/packages/wallet/wallet-toolbox/mobile/README.md @@ -13,6 +13,20 @@ Use this package in: For Node servers, use [`@bsv/wallet-toolbox`](https://www.npmjs.com/package/@bsv/wallet-toolbox). For browsers, use [`@bsv/wallet-toolbox-client`](https://www.npmjs.com/package/@bsv/wallet-toolbox-client). +## Resumable synchronization (2.15 candidate) + +`WalletStorageManager.syncFromReaderResumable(identityKey, source, options)` adds +cancellation, durable checkpoints and per-page progress. IndexedDB and Knex local +destinations yield the manager queue during source/proof I/O and between atomic +page commits. Remote destinations retain exclusive execution. Use +`maxRoughSize: 262144` as a measured starting point for constrained clients; +one oversized record may still exceed this rough page target and is subject to +the provider's separate transfer bound. Resume by invoking the API again: the +destination checkpoint is authoritative, including after a lost acknowledgement. +See the [sync contract and next-stage design](../../../../docs/guides/wallet-sync-reliability.md). +This is an eventual replica merge; it does not create a coherent source snapshot +or change the existing archive format. + ## BRC-100 result compatibility Version 2.14.1 keeps internal exact-spend accounting off public `createAction` @@ -32,6 +46,13 @@ Test the independent key and data recovery paths on a replacement device. Read [Wallet backup and recovery](https://bsv-blockchain.github.io/ts-stack/guides/wallet-backup-recovery/), [BRC-38/39 integration](https://bsv-blockchain.github.io/ts-stack/guides/wallet-data-portability/) and the [recovery checklist](https://bsv-blockchain.github.io/ts-stack/guides/wallet-recovery-drill/). +The unpublished candidate exports `RetainedReadSnapshot` and +`RetainedReadSnapshotOptions` types. They do not add a retained IndexedDB, +remote, or native mobile implementation. Unsupported local providers report +`supportsRetainedReadSnapshot() === false` and refuse `openReadSnapshot()`. +The SQL implementation and its connection/lifetime limits are described in the +[sync guide](https://bsv-blockchain.github.io/ts-stack/guides/wallet-sync-reliability/#retained-local-sql-read-views-unpublished-candidate). + Portable helpers require a concrete local `StorageProvider`; a remote client is not one. Qualify the local-copy path and device memory limits before adding export/import UI. @@ -63,6 +84,10 @@ The package publishes: The packed package is validated with Metro and compiled to optimized Hermes bytecode. Node.js 22 or newer is required for the published tooling and contributor workflow, not as an on-device runtime. +The Hermes probe uses a stable relative input filename and verifies identical +bytecode from two independent build directories. Source maps and debug data are +retained; random temporary paths must not affect the unchanged size budgets. + ### Password derivation without WebAssembly Argon2id password derivation uses `hash-wasm` when WebAssembly is available. @@ -187,6 +212,40 @@ pnpm --filter @bsv/wallet-toolbox-mobile test:mobile The gate installs the packed packages in a clean project, bundles them with Metro, checks the public export and mobile-safe module contracts, validates source maps, compiles the result with Hermes, and enforces compressed and uncompressed size budgets. +### 2.15 candidate size review and integration history + +The September 28 integration with current main measures Metro **2,401,423 / +619,596 / 468,551** bytes and Hermes **4,646,003 / 1,969,117 / 1,532,602** +bytes (raw/gzip/Brotli), within every retained ceiling. Native module composition, +source maps and Hermes compilation pass. + +The reviewed Hermes Brotli ceiling increases from 1,520,000 to 1,675,000 bytes +for bounded resumable sync, proof recovery and prepared BRC-118 payment transport. +The integrated candidate retains main’s subsequently reviewed Metro ceilings +and Hermes raw/gzip ceilings, plus this candidate’s 1,675,000-byte Hermes +Brotli ceiling. No production dependency, +minifier configuration, public export, source-map or compression setting changes +accompany this budget adjustment. + +The September 24 candidate, before current-main integration, measured with +official Node 24.18.0: + +| Artifact | Raw bytes | gzip bytes | Brotli bytes | +| -------- | --------: | ---------: | -----------: | +| Metro | 2,377,408 | 609,909 | 463,868 | +| Hermes | 4,619,927 | 1,955,135 | 1,523,206 | + +The approved Hermes Brotli ceiling gives 9.97% headroom after the final forwarding +and normalization refinements. Retaining the explicitly reviewed 1,675,000-byte ceiling +is the rationale for this small difference from the normal 10% margin. +For comparison, [upstream SDK 2.8.3 validation](https://github.com/bsv-blockchain/ts-stack/actions/runs/35948047970) +measured Metro at 2,360,475 / 602,505 / 458,183 bytes and Hermes at +4,609,755 / 1,931,907 / 1,500,120 bytes. The gate continues to inspect the installed +module graph, reject Node-only modules, validate maps and compile Hermes bytecode. +A matching-input cross-platform check produced identical Hermes bytes on Linux +x86_64 and macOS ARM, including reproduction across independent build directories. +These measurements describe this candidate fixture, not an application-size guarantee. + The reviewed 2.14.2 identity candidate `b088c1bef` measures Metro **2,379,919 / 612,262 / 462,261** bytes and Hermes **4,629,563 / 1,943,675 / 1,510,174** bytes (raw/gzip/Brotli). Compared with the preceding unpublished 2.14.2 candidate, @@ -204,3 +263,27 @@ This package is released under the [Open BSV License Version 6](./LICENSE.txt). The accompanying [THIRD_PARTY_NOTICES.md](./THIRD_PARTY_NOTICES.md) and [LICENSES/](./LICENSES/) preserve earlier Open BSV grants compiled into the mobile build. + +The 2.15 candidate also exports version-one `WalletReadSnapshot`, cursor/page and +packed-row types. The base provider explicitly refuses unsupported local pages; +only `StorageKnex` currently implements them. Type availability does not imply an +IndexedDB, native mobile or remote snapshot implementation. Existing sync and +archive APIs retain their behavior. See the [SQL paging contract](../../../../docs/guides/wallet-sync-reliability.md#profile-bound-local-sql-pages). + +The candidate also exports `snapshotSyncTables` and the version-one +`SnapshotSyncSource`, `SnapshotSyncCheckpoint`, `SnapshotSyncCommit`, +`SnapshotSyncStorage` and `SnapshotSyncTable` types. `syncToWriterResumable` adds +push progress/cancellation with an explicit exclusive fallback. These exports +support adapter integration; they do not enable SQL retention or the auxiliary +SQL migration in this browser/mobile entry point. Current IndexedDB and remote +paths retain their documented behavior. See the [local SQL integration and remaining limits](../../../../docs/guides/wallet-sync-reliability.md#durable-local-sql-sync-and-ordinary-backup). + +The candidate now includes `getSnapshotArchiveTransport(identityKey)` on +`StorageClient`. It negotiates a migrated WAL/MySQL server's immutable archive +capability and validates bounded authenticated admission/status, directories and +pages. A dedicated response ceiling applies before parsing; server and storage +identity bindings remain shared with ordinary RPC. `snapshotArchives: false` +disables the transport, and old/disabled peers decline it. This is a low-level +transport, not a retained database adapter or adoption by ordinary sync/export. +Remote row cursors, manager integration, portable semantics and physical platform +acceptance remain open. See the [archive transport contract](../../../../docs/guides/wallet-sync-reliability.md#authenticated-snapshot-archive-transport-unpublished-candidate). diff --git a/packages/wallet/wallet-toolbox/mobile/package.json b/packages/wallet/wallet-toolbox/mobile/package.json index 0e59d9bf5..fe1dbf9f6 100644 --- a/packages/wallet/wallet-toolbox/mobile/package.json +++ b/packages/wallet/wallet-toolbox/mobile/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox-mobile", - "version": "2.14.5", + "version": "2.15.0", "type": "module", "sideEffects": false, "engines": { diff --git a/packages/wallet/wallet-toolbox/mobile/platform-budget.json b/packages/wallet/wallet-toolbox/mobile/platform-budget.json index fc7b3ba68..0b7b60fb1 100644 --- a/packages/wallet/wallet-toolbox/mobile/platform-budget.json +++ b/packages/wallet/wallet-toolbox/mobile/platform-budget.json @@ -9,7 +9,7 @@ "hermes": { "raw": 5095000, "gzip": 2140000, - "brotli": 1665000 + "brotli": 1675000 } } } diff --git a/packages/wallet/wallet-toolbox/mobile/test/payment-transport.test.ts b/packages/wallet/wallet-toolbox/mobile/test/payment-transport.test.ts new file mode 100644 index 000000000..afa700a6b --- /dev/null +++ b/packages/wallet/wallet-toolbox/mobile/test/payment-transport.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, it } from 'vitest' +import { + buildMultipartPayment, + preparePaymentTransport, + resolvePaymentTransportLimits +} from '@bsv/sdk/auth/utils/paymentTransport' +import { toUTF8Strict } from '@bsv/sdk/primitives/utils' +import { WalletStorageManager } from '../../src/index.mobile' + +describe('portable sync and payment surface', () => { + it('exposes resumable sync without a Node storage backend', () => { + expect(WalletStorageManager.prototype.syncFromReaderResumable).toBeTypeOf('function') + }) + it('constructs exact multipart bytes without Buffer, FormData or text-codec globals', () => { + const names = ['Buffer', 'FormData', 'TextEncoder', 'TextDecoder'] as const + const descriptors = names.map(name => Object.getOwnPropertyDescriptor(globalThis, name)) + let body!: Uint8Array + let header!: string + try { + for (const name of names) Object.defineProperty(globalThis, name, { configurable: true, value: undefined }) + const result = buildMultipartPayment( + '{"unicode":"雪"}', + { bytes: new Uint8Array([0, 128, 255]), contentType: 'application/octet-stream' }, + 4096, + 'portable-boundary' + ) + body = result.body + header = result.contentType + const selected = preparePaymentTransport( + '{}', + { method: 'GET', headers: {} }, + new Set(['header']), + resolvePaymentTransportLimits() + ) + if (selected.transport !== 'header' || selected.headers['x-bsv-payment'] !== '{}') + throw new Error('Portable header fallback differs') + } finally { + names.forEach((name, index) => { + const descriptor = descriptors[index] + if (descriptor === undefined) Reflect.deleteProperty(globalThis, name) + else Object.defineProperty(globalThis, name, descriptor) + }) + } + expect(header).toBe('multipart/form-data; boundary=portable-boundary') + const expected = new TextEncoder().encode( + '--portable-boundary\r\nContent-Disposition: form-data; name="x-bsv-payment"\r\nContent-Type: application/json\r\n\r\n{"unicode":"雪"}\r\n--portable-boundary\r\nContent-Disposition: form-data; name="body"\r\nContent-Type: application/octet-stream\r\n\r\n' + ) + expect(body.subarray(0, expected.length)).toEqual(expected) + expect(Array.from(body.subarray(expected.length, expected.length + 3))).toEqual([0, 128, 255]) + expect(toUTF8Strict(body.subarray(expected.length + 3))).toBe('\r\n--portable-boundary--\r\n') + }) +}) diff --git a/packages/wallet/wallet-toolbox/package.json b/packages/wallet/wallet-toolbox/package.json index 241504607..400e3b22d 100644 --- a/packages/wallet/wallet-toolbox/package.json +++ b/packages/wallet/wallet-toolbox/package.json @@ -1,6 +1,6 @@ { "name": "@bsv/wallet-toolbox", - "version": "2.14.5", + "version": "2.15.0", "sideEffects": false, "type": "commonjs", "engines": { @@ -55,7 +55,7 @@ "test": "pnpm build && jest --testPathIgnorePatterns='man.test.ts|live.test.ts|bench.test.ts|client/test|mobile/test' --watchman=false", "test:watch": "pnpm build && jest --testPathIgnorePatterns='man.test.ts|live.test.ts|client/test|mobile/test' --watch", "test:coverage": "pnpm build && jest --testPathIgnorePatterns='man.test.ts|live.test.ts|bench.test.ts|client/test|mobile/test' --coverage --watchman=false", - "test:property": "jest --runInBand --watchman=false --runTestsByPath src/utility/__tests/actionBatchPack.property.test.ts --testPathIgnorePatterns=man.test.ts", + "test:property": "jest --runInBand --watchman=false --runTestsByPath src/utility/__tests/actionBatchPack.property.test.ts src/storage/snapshot/RetainedReadSnapshot.property.test.ts src/storage/snapshot/SnapshotSync.property.test.ts src/storage/snapshot/SnapshotSyncDestination.property.test.ts src/storage/snapshot/SnapshotSyncRows.property.test.ts src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts src/storage/snapshot/archive/SnapshotArchiveProtocol.property.test.ts src/storage/snapshot/archive/RemoteSnapshotReader.property.test.ts src/storage/snapshot/journal/SnapshotJournal.property.test.ts src/storage/portable/mysqlReadSnapshot.property.test.ts --testPathIgnorePatterns=man.test.ts", "test:manual": "pnpm build && node ../../../scripts/run-governed-test.mjs manual", "test:live": "pnpm build && node ../../../scripts/run-governed-test.mjs live", "operator:build": "pnpm build && tsc --project operator/tsconfig.json --pretty false", @@ -63,16 +63,20 @@ "bench:action-batch": "pnpm build && jest --runInBand --runTestsByPath benchmarks/action-batch.bench.test.ts --testPathIgnorePatterns=man.test.ts", "bench:create-action-funding": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-funding.bench.test.ts --testPathIgnorePatterns=man.test.ts", "bench:create-action-beef": "pnpm build && jest --runInBand --runTestsByPath benchmarks/create-action-beef.bench.test.ts --testPathIgnorePatterns=man.test.ts", - "bench:storage-sync": "pnpm build && jest --runInBand --runTestsByPath benchmarks/storage-sync.bench.test.ts --testPathIgnorePatterns=man.test.ts", + "bench:storage-sync": "pnpm build && jest --runInBand --runTestsByPath benchmarks/storage-sync.bench.test.ts benchmarks/resumable-sync.bench.test.ts --testPathIgnorePatterns=man.test.ts", "format:check": "pnpm --workspace-root exec prettier --check \"packages/wallet/wallet-toolbox/{README.md,jest.config.cjs,package.json,tsconfig*.json}\"", "lint": "oxlint src test benchmarks examples operator --deny-warnings", "lint:ci": "pnpm lint", - "pack:check": "pnpm build && node ../../../scripts/check-package-artifact.mjs . --modes cjs --exports Wallet,WalletSigner,WalletStorageManager,StorageClient,StorageKnex,Services,Setup,WalletPermissionsManager,WalletSettingsManager,LocalChainTracker,FixedWindowBulkFileDownloadBudget,DurableFileBulkFileDownloadBudget,BulkFileDataCacheFs,NodeBulkFileDataValidator,sdk --entry-exports \"./out/src/sdk=WalletError|WERR_BAD_REQUEST|WERR_INTERNAL|WERR_INVALID_PARAMETER|WERR_UNAUTHORIZED\"", + "pack:check": "pnpm build && node ../../../scripts/check-package-artifact.mjs . --modes cjs --exports Wallet,WalletSigner,WalletStorageManager,StorageClient,StorageKnex,Services,Setup,WalletPermissionsManager,WalletSettingsManager,LocalChainTracker,FixedWindowBulkFileDownloadBudget,DurableFileBulkFileDownloadBudget,BulkFileDataCacheFs,NodeBulkFileDataValidator,sdk --entry-exports \"./out/src/sdk=WalletError|WERR_BAD_REQUEST|WERR_INTERNAL|WERR_INVALID_PARAMETER|WERR_UNAUTHORIZED\" && tsc --project test/consumer/tsconfig.json", "typecheck": "tsc --build --pretty false && tsc --project examples/tsconfig.json --pretty false && tsc --project operator/tsconfig.json --noEmit --pretty false", "build": "tsc --build", "prepublishOnly": "pnpm build", "doc": "ts2md", - "sync-versions": "node syncVersions.js" + "sync-versions": "node syncVersions.js", + "test:snapshot-archive-crash": "pnpm build && node test/storage/snapshotArchiveCrash.cjs", + "test:snapshot-archive-mysql": "pnpm build && node test/storage/runSnapshotArchiveMysql.cjs", + "test:snapshot-journal-crash": "pnpm build && node test/storage/snapshotJournalSqliteCrash.cjs", + "test:snapshot-journal-mysql": "pnpm build && node test/storage/runSnapshotJournalMysql.cjs" }, "bugs": { "url": "https://github.com/bsv-blockchain/ts-stack/issues" diff --git a/packages/wallet/wallet-toolbox/src/sdk/ActionBatch.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/ActionBatch.interfaces.ts index 650e6f474..cc2a6bf12 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/ActionBatch.interfaces.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/ActionBatch.interfaces.ts @@ -11,6 +11,13 @@ export type ActionBatchPackEncoding = 'identity' | 'gzip' | 'brotli' /** Internal Wallet Toolbox capabilities. These do not extend the BRC-100 wallet interface. */ export interface StorageCapabilities { + /** Opt-in manager scheduling guarantees; absence retains exclusive access. */ + storageAccess?: { + version: 1 + concurrentReads: boolean + /** Page data and its compare-and-set checkpoint commit in one transaction. */ + atomicSyncPages: boolean + } /** Built-in BRC-177 pre-funding, durable expiry, and reclaim support. */ brc177NoSendExpiry?: { version: 1 diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletErrorFromJson.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletErrorFromJson.ts index 88fac31e9..6d57b4d37 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/WalletErrorFromJson.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/WalletErrorFromJson.ts @@ -8,6 +8,7 @@ import { WERR_INSUFFICIENT_FUNDS, WERR_INTERNAL, WERR_INVALID_OPERATION, + WERR_INVALID_MERKLE_ROOT, WERR_INVALID_PARAMETER, WERR_INVALID_PUBLIC_KEY, WERR_MISSING_PARAMETER, @@ -189,6 +190,18 @@ export function WalletErrorFromJson(json: object): WalletError { case 'WERR_NETWORK_CHAIN': e = new WERR_NETWORK_CHAIN(obj.message) break + case 'WERR_INVALID_MERKLE_ROOT': + if (!Number.isSafeInteger(obj.blockHeight) || obj.blockHeight < 0) { + throw new WERR_INTERNAL('Invalid remote wallet error blockHeight') + } + e = new WERR_INVALID_MERKLE_ROOT( + boundedString(obj.blockHash, 'blockHash', 64), + obj.blockHeight, + boundedString(obj.merkleRoot, 'merkleRoot', 64), + obj.txid === undefined ? undefined : boundedString(obj.txid, 'txid', 64) + ) + if (obj.message !== undefined) e.message = boundedString(obj.message, 'message') + break case 'WERR_UNAUTHORIZED': e = new WERR_UNAUTHORIZED(obj.message) break diff --git a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts index 646393729..fa07a44f9 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/WalletStorage.interfaces.ts @@ -38,6 +38,7 @@ import { import { WalletServices } from './WalletServices.interfaces' import { Chain, Paged, ProvenTxReqStatus, TransactionStatus } from './types' import { WalletError } from './WalletError' +import type { SnapshotSyncStorage } from '../storage/snapshot/SnapshotSync' import { AbortActionBatchResult, ActionBatchManifest, @@ -81,7 +82,7 @@ export interface WalletStorage { getAuth: () => Promise - findOrInsertUser: (identityKey: string) => Promise<{ user: TableUser, isNew: boolean }> + findOrInsertUser: (identityKey: string) => Promise<{ user: TableUser; isNew: boolean }> abortAction: (args: AbortActionArgs) => Promise createAction: (args: Validation.ValidCreateActionArgs) => Promise @@ -146,7 +147,7 @@ export interface WalletStorageProvider extends WalletStorageSync { setServices: (v: WalletServices) => void } -export interface WalletStorageSync extends WalletStorageWriter { +export interface WalletStorageSync extends WalletStorageWriter, WalletStorageSyncReader { /** Compact writer checkpoint. Undefined means a legacy remote provider. */ getSyncCheckpoint?: ( auth: AuthId, @@ -158,7 +159,7 @@ export interface WalletStorageSync extends WalletStorageWriter { auth: AuthId, storageIdentityKey: string, storageName: string - ) => Promise<{ syncState: TableSyncState, isNew: boolean }> + ) => Promise<{ syncState: TableSyncState; isNew: boolean }> /** * Updagte the `activeStorage` property of the authenticated user by their `userId`. @@ -177,6 +178,8 @@ export interface WalletStorageSync extends WalletStorageWriter { export interface WalletStorageSyncReader { makeAvailable: () => Promise getSyncChunk: (args: RequestSyncChunkArgs) => Promise + /** Optional local adapter; immutable remote sources negotiate their capability when opened. */ + getSnapshotSync?: () => SnapshotSyncStorage | undefined } export interface WalletStorageWriter extends WalletStorageReader { @@ -184,7 +187,7 @@ export interface WalletStorageWriter extends WalletStorageReader { migrate: (storageName: string, storageIdentityKey: string) => Promise destroy: () => Promise - findOrInsertUser: (identityKey: string) => Promise<{ user: TableUser, isNew: boolean }> + findOrInsertUser: (identityKey: string) => Promise<{ user: TableUser; isNew: boolean }> abortAction: (auth: AuthId, args: AbortActionArgs) => Promise createAction: (auth: AuthId, args: Validation.ValidCreateActionArgs) => Promise @@ -207,10 +210,7 @@ export interface WalletStorageWriter extends WalletStorageReader { putActionBatchBlob: (auth: AuthId, args: PutActionBatchBlobArgs) => Promise putActionBatchPack?: (auth: AuthId, args: PutActionBatchPackArgs) => Promise commitActionBatch: (auth: AuthId, manifest: ActionBatchManifest) => Promise - commitActionBatchByDigest?: ( - auth: AuthId, - args: CommitActionBatchByDigestArgs - ) => Promise + commitActionBatchByDigest?: (auth: AuthId, args: CommitActionBatchByDigestArgs) => Promise abortActionBatch: (auth: AuthId, batchId: string) => Promise internalizeAction: (auth: AuthId, args: InternalizeActionArgs) => Promise @@ -590,7 +590,7 @@ export type SyncProtocolVersion = '0.1.0' export interface SyncCheckpoint { syncStateId: number since?: Date - offsets: Array<{ name: string, offset: number }> + offsets: Array<{ name: string; offset: number }> } export interface RequestSyncChunkArgs { @@ -656,7 +656,7 @@ export interface RequestSyncChunkArgs { * 10 Certificates * 11 CertificateFields */ - offsets: Array<{ name: string, offset: number }> + offsets: Array<{ name: string; offset: number }> } export interface SyncChunkTotals { @@ -715,6 +715,11 @@ export interface ProcessSyncChunkResult { maxUpdated_at: Date | undefined updates: number inserts: number + /** + * Optional failure channel for custom providers; local providers normally throw. + * A failure takes precedence over done, counters and nextCheckpoint. Callers stop + * and resume from the destination's durable checkpoint instead of retrying a write. + */ error?: WalletError } @@ -729,7 +734,7 @@ export interface ReproveHeaderResult { /** * List of proven_txs records that were updated with new proof data. */ - updated: Array<{ was: TableProvenTx, update: Partial, logUpdate: string }> + updated: Array<{ was: TableProvenTx; update: Partial; logUpdate: string }> /** * List of proven_txs records that were checked but currently available proof is unchanged. */ @@ -751,7 +756,7 @@ export interface ReproveProvenResult { /** * Valid if proof data for proven_txs record is available and has changed. */ - updated?: { update: Partial, logUpdate: string } + updated?: { update: Partial; logUpdate: string } /** * True if proof data for proven_txs record was found to be unchanged. */ diff --git a/packages/wallet/wallet-toolbox/src/sdk/__test/WalletError.test.ts b/packages/wallet/wallet-toolbox/src/sdk/__test/WalletError.test.ts index 5035b5c60..ae8c9f25d 100644 --- a/packages/wallet/wallet-toolbox/src/sdk/__test/WalletError.test.ts +++ b/packages/wallet/wallet-toolbox/src/sdk/__test/WalletError.test.ts @@ -9,6 +9,7 @@ import { WERR_INSUFFICIENT_FUNDS, WERR_BROADCAST_UNAVAILABLE, WERR_NETWORK_CHAIN, + WERR_INVALID_MERKLE_ROOT, WERR_INVALID_OPERATION, WERR_MISSING_PARAMETER, WERR_BAD_REQUEST, @@ -436,3 +437,26 @@ describe('WalletError tests', () => { }) }) }) + +test('preserves typed proof failure and recovery context across remote JSON', () => { + const error = new WERR_INVALID_MERKLE_ROOT('11'.repeat(32), 100, '22'.repeat(32), '33'.repeat(32)) + error.message += ' Retry canonical recovery.' + const result = WalletErrorFromJson(JSON.parse(error.toJson())) + expect(result).toBeInstanceOf(WERR_INVALID_MERKLE_ROOT) + expect(result.toJson()).toEqual(error.toJson()) + for (const changes of [ + { blockHeight: -1 }, + { blockHeight: NaN }, + { blockHeight: 1.5 }, + { blockHash: {} }, + { merkleRoot: 'x'.repeat(65) }, + { txid: [] }, + { message: 'x'.repeat(4097) } + ]) { + expect(() => WalletErrorFromJson({ ...JSON.parse(error.toJson()), ...changes })).toThrow( + 'Invalid remote wallet error' + ) + } + const noTxid = new WERR_INVALID_MERKLE_ROOT('11'.repeat(32), 0, '22'.repeat(32)) + expect(WalletErrorFromJson(JSON.parse(noTxid.toJson())).toJson()).toBe(noTxid.toJson()) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts index 5251d16d7..3232bc86f 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageIdb.ts @@ -1,3 +1,4 @@ +import { recoveredProofUpdate, sameSyncProof } from './methods/validateSyncProof' import { type ValidListActionsArgs, type ValidListOutputsArgs } from '@bsv/sdk/wallet/validationHelpers' import { deleteDB, IDBPDatabase, IDBPObjectStore, IDBPTransaction, openDB } from 'idb' import { @@ -139,6 +140,10 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider return true } + protected override supportsStorageAccessScheduling(): boolean { + return true + } + protected override supportsNoSendExpiryPersistence(): boolean { return true } @@ -212,8 +217,13 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider * * @param trx */ - async readSettings(_trx?: TrxToken): Promise { + async readSettings(trx?: TrxToken): Promise { await this.verifyDB() + if (trx != null) { + const rows = await this.toDbTrx(['settings'], 'readonly', trx).objectStore('settings').getAll() + if (rows.length !== 1) throw new WERR_INTERNAL('Wallet snapshot requires exactly one settings row') + return this.validateEntity(rows[0]) + } return this._settings as TableSettings } @@ -1236,6 +1246,25 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider return await this.updateIdb(id, update, 'provenTxId', 'proven_txs', trx) } + override async compareAndSetProvenTxProof( + expected: TableProvenTx, + replacement: TableProvenTx, + trx?: TrxToken + ): Promise { + const update = recoveredProofUpdate(expected, replacement) + const dbTrx = this.toDbTrx(['proven_txs'], 'readwrite', trx) + const store = dbTrx.objectStore('proven_txs') + try { + const current = await store.get(expected.provenTxId) + if (current == null || !sameSyncProof(current, expected)) return false + await (store.put as (value: TableProvenTx) => Promise)({ ...current, ...update }) + this.isDirty = true + return true + } finally { + if (trx == null) await dbTrx.done + } + } + async updateProvenTxReq(id: number | number[], update: Partial, trx?: TrxToken): Promise { return await this.updateIdb(id, update, 'provenTxReqId', 'proven_tx_reqs', trx) } @@ -1522,11 +1551,33 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider if (trx == null) await tx.done } - /** - * @param scope - * @param trx - * @returns - */ + override supportsReadSnapshot(): boolean { + return true + } + + /** Capture settings and wallet tables in one local readonly view. */ + override async readSnapshot(read: (trx: TrxToken) => Promise): Promise { + const db = await this.verifyDB() + // One readonly transaction gives all wallet tables the same view. This + // permits other reads; IndexedDB serializes overlapping writers until + // capture ends. No peer I/O or consumer callback belongs in this scope. + const tx = db.transaction(['settings', ...this.allStores], 'readonly') + try { + const result = await read(tx as TrxToken) + await tx.done + return result + } catch (error) { + try { + tx.abort() + await tx.done + } catch { + // An already-aborted/finished transaction needs no further cleanup; + // preserve the original capture error rather than its abort result. + } + throw error + } + } + async transaction(scope: (trx: TrxToken) => Promise, trx?: TrxToken): Promise { if (trx != null) return await scope(trx) @@ -2091,10 +2142,21 @@ export class StorageIdb extends StorageProvider implements WalletStorageProvider } else { cursor = await store.openCursor() } + let offset = args.paged?.offset ?? 0 + // A full user-label index scan already enforces its entire predicate. Skip + // prior pages in IndexedDB rather than cloning and filtering every prior + // row for every source query. Keep the filtered/since path unchanged. + const indexedOnly = + args.since == null && + Object.entries(args.partial ?? {}).every(([key, value]) => value === undefined || key === 'userId') + if (cursor != null && indexedOnly && Number.isSafeInteger(offset) && offset > 0) { + cursor = await cursor.advance(offset) + offset = 0 + } await scanCursor( cursor, args.since, - args.paged?.offset ?? 0, + offset, args.paged?.limit, r => matchesTxLabelPartial(r, args.partial), filtered diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts index 7b7436e08..501f7cdef 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageKnex.ts @@ -1,3 +1,41 @@ +import { + maintainSnapshotJournal as startSnapshotJournalMaintenance, + type SnapshotJournalMaintenanceRequest, + type SnapshotJournalMaintenanceResult +} from './snapshot/journal/SnapshotJournalMaintenance' +import type { + SnapshotJournalMaintenanceTask, + SnapshotJournalMaintenanceOptions +} from './snapshot/journal/SnapshotJournalMaintenanceTask' +import { + retainSnapshotJournalCapture, + type SnapshotJournalCaptureRequest, + type SnapshotJournalCaptureLifetime, + type SnapshotJournalSource +} from './snapshot/journal/SnapshotJournalCapture' +import { SnapshotJournalConnectionCleanupError } from './snapshot/journal/SnapshotJournalConnections' +import { dropGenerationForDataDeletion } from './schema/snapshotSqliteIndexMigration' +import { migration as SNAPSHOT_SQLITE_INDEX_MIGRATION } from './schema/snapshotSqliteIndexState' +import { SnapshotResourceLimitError } from './snapshot/SnapshotResourceLimitError' +import { readGuardedSnapshotArchive, recoverSnapshotArchiveGuards } from './snapshot/archive/SnapshotArchiveGuard' +import type { SnapshotArchiveRequestOwner } from './snapshot/archive/SnapshotArchiveRequest' +import { SnapshotCancelledError } from './snapshot/SnapshotCancelledError' +import type { SnapshotSyncStorage } from './snapshot/SnapshotSync' +import { KnexSnapshotSyncDestination } from './snapshot/KnexSnapshotSyncDestination' +import type { WalletReadSnapshot, WalletReadSnapshotOptions } from './snapshot/WalletReadSnapshot' +import { openKnexWalletReadSnapshot } from './snapshot/KnexWalletReadSnapshot' +import { + openKnexSnapshotArchiveSource, + SnapshotArchiveSourceCleanupError, + type SnapshotArchiveSource +} from './snapshot/archive/KnexSnapshotArchiveSource' +import { + retainReadSnapshot, + type RetainedReadSnapshot, + type RetainedReadSnapshotLifetime, + type RetainedReadSnapshotOptions +} from './snapshot/RetainedReadSnapshot' +import { recoveredProofUpdate } from './methods/validateSyncProof' import { type ValidListActionsArgs, type ValidListOutputsArgs } from '@bsv/sdk/wallet/validationHelpers' import { ListActionsResult, ListOutputsResult, TelemetrySpan } from '@bsv/sdk' import { @@ -24,7 +62,7 @@ import { import { TableActionBatch, TableActionBatchBlob, TableActionBatchOutput } from './schema/tables/TableActionBatch' import { TablePreparedBeef } from './schema/tables/TablePreparedBeef.interfaces' import { KnexMigrations } from './schema/KnexMigrations' -import { Knex } from 'knex' +import { knex as createKnex, Knex } from 'knex' import { AdminStatsResult, StorageProvider, StorageProviderOptions } from './StorageProvider' import { purgeData } from './methods/purgeData' import { listActions } from './methods/listActionsKnex' @@ -60,7 +98,13 @@ import { SyncChunkTotals, WalletStorageProvider } from '../sdk/WalletStorage.interfaces' -import { WERR_INTERNAL, WERR_INVALID_PARAMETER, WERR_NOT_IMPLEMENTED, WERR_UNAUTHORIZED } from '../sdk/WERR_errors' +import { + WERR_INTERNAL, + WERR_INVALID_OPERATION, + WERR_INVALID_PARAMETER, + WERR_NOT_IMPLEMENTED, + WERR_UNAUTHORIZED +} from '../sdk/WERR_errors' import { verifyId, verifyOne, verifyOneOrNone } from '../utility/utilityHelpers' import { EntityTimeStamp, TransactionStatus } from '../sdk/types' @@ -86,6 +130,8 @@ export interface StorageKnexOptions extends StorageProviderOptions { knex: Knex /** Optional prepared-BEEF (COOK) rollout controls. Disabled by default. */ preparedBeef?: PreparedBeefOptions + /** Default true. False keeps legacy sync scheduling while retaining the additive schema for forward rollback. */ + snapshotSync?: boolean } // Keep bulk statements below conservative SQLite/MySQL bind-parameter @@ -93,6 +139,20 @@ export interface StorageKnexOptions extends StorageProviderOptions { const ACTION_BATCH_BLOB_SQL_CHUNK = 500 const OUTPUT_INSERT_SQL_CHUNK = 500 +// Only provider-owned snapshot callbacks can classify a transaction as a read +// view. Keep this private and shared across providers that use the same token. +const readSnapshotTransactions = new WeakSet() + +async function withinReadSnapshot(trx: TrxToken, read: (trx: TrxToken) => Promise): Promise { + const alreadyTracked = readSnapshotTransactions.has(trx) + readSnapshotTransactions.add(trx) + try { + return await read(trx) + } finally { + if (!alreadyTracked) readSnapshotTransactions.delete(trx) + } +} + interface KnexTelemetryQuery { __knexQueryUid?: string method?: string @@ -105,6 +165,19 @@ interface PreparedBeefMetadata { export class StorageKnex extends StorageProvider implements WalletStorageProvider { knex: Knex + private readonly snapshotSyncEnabled: boolean + private readonly snapshotSyncTelemetry: StorageKnexOptions['telemetry'] + private snapshotSyncSource?: StorageKnex + private snapshotSyncOpening?: Promise + private snapshotSyncBusy = false + private snapshotJournalCapture?: SnapshotJournalCaptureLifetime + private snapshotJournalCaptureFailure?: SnapshotJournalConnectionCleanupError + private snapshotJournalMaintenance?: SnapshotJournalMaintenanceTask + private snapshotJournalMaintenanceFailure?: SnapshotJournalConnectionCleanupError + private snapshotArchiveRecovery?: Promise + private guardedSnapshotFailure?: { error: unknown } + private retainedReadSnapshot?: RetainedReadSnapshotLifetime + private retainedReadSnapshotsStopped = false readonly preparedBeefPolicy: PreparedBeefPolicy private readonly preparedBeefCoordinator: PreparedBeefCoordinator private readonly preparedBeefReadSuspensions = new Set() @@ -136,6 +209,9 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide if (options.knex == null) throw new WERR_INVALID_PARAMETER('options.knex', 'valid') this.knex = options.knex parsePostgresInt8AsNumber(this.knex) + this.snapshotSyncEnabled = options.snapshotSync ?? true + this.snapshotSyncTelemetry = options.telemetry + if (typeof this.snapshotSyncEnabled !== 'boolean') throw new WERR_INVALID_PARAMETER('snapshotSync', 'a boolean') this.preparedBeefPolicy = validatePreparedBeefPolicy(options.preparedBeef) this.preparedBeefCoordinator = new PreparedBeefCoordinator(this) if (this.telemetry.enabled) { @@ -164,6 +240,445 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide protected override supportsActionBatchPersistence(): boolean { return true } + + protected override supportsStorageAccessScheduling(): boolean { + return true + } + + override supportsReadSnapshot(): boolean { + return this.databaseSystem() === 'sqlite' || this.databaseSystem() === 'mysql' + } + + override async readSnapshot(read: (trx: TrxToken) => Promise): Promise { + await this.makeAvailable() + const database = this.databaseSystem() + if (database === 'sqlite') { + // SQLite establishes its read view on the first query. Do not request + // Knex's unsupported SQLite isolation/readOnly options or write settings + // into the shared connection. WAL writers may use another connection. + return await this.knex.transaction(trx => withinReadSnapshot(trx, read)) + } + if (database === 'mysql') { + return await this.readMySQLSnapshot(trx => withinReadSnapshot(trx, read)) + } + throw new WERR_NOT_IMPLEMENTED('Coherent wallet source snapshots require SQLite or MySQL isolation') + } + + override supportsRetainedReadSnapshot(): boolean { + return this.supportsReadSnapshot() + } + + /** One retained local transaction per provider; it occupies one pool connection until physical cleanup. */ + override async openReadSnapshot(options: RetainedReadSnapshotOptions = {}): Promise { + return await this.openTrackedReadSnapshot(options, read => this.readSnapshot(read)) + } + + private async openTrackedReadSnapshot( + options: RetainedReadSnapshotOptions, + transaction: (read: (trx: TrxToken) => Promise) => Promise + ): Promise { + if (this.retainedReadSnapshotsStopped) { + throw new WERR_INVALID_OPERATION('Retained read snapshots are unavailable after provider destruction begins') + } + if (!this.supportsRetainedReadSnapshot()) return await super.openReadSnapshot(options) + if (this.retainedReadSnapshot !== undefined) { + throw new WERR_INVALID_OPERATION('This provider already has a retained read snapshot opening or active') + } + const lifetime = retainReadSnapshot( + read => transaction(trx => withinReadSnapshot(trx, read)), + async trx => { + // Pin SQLite's deferred read view before opening resolves. MySQL also + // establishes its repeatable-read snapshot on this first data read. + await this.readSettings(trx) + }, + options + ) + this.retainedReadSnapshot = lifetime + const release = (): void => { + if (this.retainedReadSnapshot === lifetime) this.retainedReadSnapshot = undefined + } + void lifetime.closed.then(release, release) + return await lifetime.opened + } + + private async stopRetainedReadSnapshots(): Promise { + // Fence admission before any asynchronous cleanup can yield. A view whose + // close releases the capacity slot must not permit reopening during destroy. + this.retainedReadSnapshotsStopped = true + try { + await this.snapshotArchiveRecovery + } finally { + try { + await this.snapshotJournalCapture?.close() + } finally { + try { + await this.snapshotJournalMaintenance?.close() + } finally { + await this.snapshotSyncOpening?.catch(() => undefined) + await this.snapshotSyncSource?.destroy() + await this.retainedReadSnapshot?.close() + } + } + } + } + + override supportsWalletReadSnapshot(): boolean { + return this.supportsRetainedReadSnapshot() + } + + override async openWalletReadSnapshot( + identityKey: string, + options: WalletReadSnapshotOptions = {} + ): Promise { + if (!this.supportsWalletReadSnapshot()) return await super.openWalletReadSnapshot(identityKey, options) + return await openKnexWalletReadSnapshot(this, identityKey, options) + } + + private async readMySQLSnapshot(read: (trx: TrxToken) => Promise): Promise { + const client = this.knex.client + const connection = await client.acquireConnection() + try { + // MySQL requires a comma between transaction characteristics. Knex's + // combined isolationLevel/readOnly options currently omit it. Reserve + // one connection, set only its NEXT transaction, then let Knex own the + // begin/commit/rollback lifecycle without changing pool session defaults. + await this.knex.raw('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY').connection(connection) + return await this.knex.transaction( + async trx => { + if (trx.isCompleted()) throw new WERR_INTERNAL('MySQL snapshot transaction did not begin') + return await read(trx) + }, + { connection } + ) + } catch (error) { + // A setup/begin failure can leave pending transaction characteristics. + // Closing before release prevents a later wallet write inheriting them. + // Driver destroy also removes native mysql2 pooled connections; end() + // may merely return those connections to their external pool. + connection.destroy() + throw error + } finally { + delete connection.__knexTxId + await client.releaseConnection(connection) + } + } + + override getSnapshotSync(): SnapshotSyncStorage | undefined { + if (!this.snapshotSyncEnabled || !this.supportsWalletReadSnapshot()) return undefined + const destination = new KnexSnapshotSyncDestination(this, chunk => this.prepareSyncProofs(chunk)) + return { + supportsDestination: () => destination.supportsDestination(), + openSource: (identityKey, options) => + this.openConcurrentSyncSource(identityKey, options ?? {}, (reader, identity, settings) => + reader.openWalletReadSnapshot(identity, settings) + ), + begin: (source, activeStorage) => destination.begin(source, activeStorage), + checkpoint: (identityKey, sourceIdentity) => destination.checkpoint(identityKey, sourceIdentity), + prepare: (checkpoint, page) => destination.prepare(checkpoint, page) + } + } + + /** Local capability probe; no reader pool is constructed or RPC advertised. */ + async supportsSnapshotArchiveSource(): Promise { + if (this.retainedReadSnapshotsStopped) return false + const config = await this.concurrentSnapshotReaderConfig() + return ( + !this.retainedReadSnapshotsStopped && + config !== undefined && + (config.client === 'better-sqlite3' || String(config.client).includes('mysql')) + ) + } + + /** Shares local sync's single owned reader slot and awaited physical cleanup. */ + async openSnapshotArchiveSource( + identityKey: string, + options: WalletReadSnapshotOptions = {}, + owner?: SnapshotArchiveRequestOwner + ): Promise { + const claim = owner === undefined ? undefined : { ...owner } + try { + return await this.openConcurrentSyncSource(identityKey, options, async (reader, identity, settings) => { + if (claim === undefined) return await openKnexSnapshotArchiveSource(reader, identity, settings) + return await openKnexSnapshotArchiveSource(reader, identity, settings, () => + reader.openTrackedReadSnapshot(settings, async read => { + try { + await reader.makeAvailable() + await readGuardedSnapshotArchive(this.knex, reader.knex, claim, read) + } catch (error) { + reader.guardedSnapshotFailure = { error } + throw error + } + }) + ) + }) + } catch (error) { + if (error instanceof SnapshotArchiveSourceCleanupError) throw error + if (this.retainedReadSnapshotsStopped && this.snapshotSyncSource !== undefined) + throw new SnapshotArchiveSourceCleanupError(error) + throw error + } + } + + /** Internal complete-generation capture; no RPC or incremental capability is advertised. + * The source shares sync/archive admission and retains it through physical cleanup. + */ + async openSnapshotJournalSource( + identityKey: string, + request: SnapshotJournalCaptureRequest, + options: RetainedReadSnapshotOptions = {} + ): Promise { + if (this.retainedReadSnapshotsStopped) + throw new WERR_INVALID_OPERATION('Snapshot journal sources are unavailable after destruction begins') + if (this.snapshotSyncBusy) + throw new WERR_INVALID_OPERATION('This provider already has a snapshot sync source opening or active') + this.snapshotSyncBusy = true + let lifetime: SnapshotJournalCaptureLifetime + try { + lifetime = retainSnapshotJournalCapture( + this.chain, + async () => { + if (this.retainedReadSnapshotsStopped) + throw new WERR_INVALID_OPERATION('Snapshot journal sources are unavailable after destruction begins') + return await this.concurrentSnapshotReaderConfig() + }, + identityKey, + request, + options + ) + } catch (error) { + this.snapshotSyncBusy = false + throw error + } + this.snapshotJournalCapture = lifetime + const release = (error?: unknown): void => { + if (error instanceof SnapshotJournalConnectionCleanupError) { + this.snapshotJournalCaptureFailure = error + this.retainedReadSnapshotsStopped = true + return + } + if (this.snapshotJournalCapture === lifetime) { + this.snapshotJournalCapture = undefined + this.snapshotSyncBusy = false + } + } + void lifetime.closed.then(() => release(), release) + return await lifetime.opened + } + + /** One internal global floor/page operation; shares source admission until native cleanup drains. */ + async maintainSnapshotJournal( + request: SnapshotJournalMaintenanceRequest, + options: SnapshotJournalMaintenanceOptions = {} + ): Promise { + if (this.retainedReadSnapshotsStopped) + throw new WERR_INVALID_OPERATION('Snapshot journal maintenance is unavailable after destruction begins') + if (this.snapshotSyncBusy) + throw new WERR_INVALID_OPERATION('This provider already has a snapshot source or maintenance opening or active') + this.snapshotSyncBusy = true + let lifetime: SnapshotJournalMaintenanceTask + try { + lifetime = startSnapshotJournalMaintenance( + async () => { + if (this.retainedReadSnapshotsStopped) + throw new WERR_INVALID_OPERATION('Snapshot journal maintenance is unavailable after destruction begins') + return await this.concurrentSnapshotReaderConfig() + }, + request, + options + ) + } catch (error) { + this.snapshotSyncBusy = false + throw error + } + this.snapshotJournalMaintenance = lifetime + const release = (error?: unknown): void => { + if (error instanceof SnapshotJournalConnectionCleanupError) { + this.snapshotJournalMaintenanceFailure = error + this.retainedReadSnapshotsStopped = true + return + } + if (this.snapshotJournalMaintenance === lifetime) { + this.snapshotJournalMaintenance = undefined + this.snapshotSyncBusy = false + } + } + void lifetime.closed.then(() => release(), release) + return await lifetime.result + } + + /** Drain an already-stopping maintenance owner; a failed native cleanup remains observable. */ + awaitSnapshotJournalMaintenanceCleanup(): Promise { + if (this.snapshotJournalMaintenanceFailure !== undefined) + return Promise.reject(this.snapshotJournalMaintenanceFailure) + return this.snapshotJournalMaintenance?.closed ?? Promise.resolve() + } + + /** Drain an already-stopping capture without admitting another source. */ + awaitSnapshotJournalCaptureCleanup(): Promise { + if (this.snapshotJournalCaptureFailure !== undefined) return Promise.reject(this.snapshotJournalCaptureFailure) + return this.snapshotJournalCapture?.closed ?? Promise.resolve() + } + + /** One bounded recovery flight per provider, drained by provider destruction. */ + recoverSnapshotArchiveSources(): Promise { + if (this.retainedReadSnapshotsStopped) + return Promise.reject(new WERR_INVALID_OPERATION('Snapshot source recovery is unavailable after destruction')) + if (this.snapshotArchiveRecovery !== undefined) return this.snapshotArchiveRecovery + const recovery = Promise.resolve() + .then(async () => { + const config = await this.concurrentSnapshotReaderConfig() + if (config === undefined) return + await recoverSnapshotArchiveGuards(this.knex, config) + }) + .finally(() => { + if (this.snapshotArchiveRecovery === recovery) this.snapshotArchiveRecovery = undefined + }) + this.snapshotArchiveRecovery = recovery + return recovery + } + + /** Drain an already-started recovery without admitting another one during service shutdown. */ + awaitSnapshotArchiveRecovery(): Promise { + return this.snapshotArchiveRecovery ?? Promise.resolve() + } + + private async openConcurrentSyncSource( + identityKey: string, + options: WalletReadSnapshotOptions, + openSource: (reader: StorageKnex, identityKey: string, options: WalletReadSnapshotOptions) => Promise + ): Promise { + if (this.retainedReadSnapshotsStopped) + throw new WERR_INVALID_OPERATION('Snapshot sync is unavailable after destruction begins') + if (this.snapshotSyncBusy) + throw new WERR_INVALID_OPERATION('This provider already has a snapshot sync source opening or active') + const lifetimeMs = options.lifetimeMs ?? 300000 + if (!Number.isSafeInteger(lifetimeMs) || lifetimeMs < 1 || lifetimeMs > 3600000) { + throw new WERR_INVALID_PARAMETER('lifetimeMs', 'an integer from 1 to 3600000') + } + const deadline = { + expiresAt: Date.now() + lifetimeMs, + startedAt: performance.now(), + lifetimeMs + } + this.snapshotSyncBusy = true + const opening: Promise = this.createConcurrentSyncSource( + identityKey, + { ...options }, + deadline, + openSource + ) + .then( + view => { + if (view === undefined) this.snapshotSyncBusy = false + return view + }, + error => { + this.snapshotSyncBusy = false + throw error + } + ) + .finally(() => { + if (this.snapshotSyncOpening === opening) this.snapshotSyncOpening = undefined + }) + this.snapshotSyncOpening = opening + return await opening + } + + private async createConcurrentSyncSource( + identityKey: string, + options: WalletReadSnapshotOptions, + deadline: { expiresAt: number; startedAt: number; lifetimeMs: number }, + openSource: (reader: StorageKnex, identityKey: string, options: WalletReadSnapshotOptions) => Promise + ): Promise { + if (options.signal?.aborted === true) throw new SnapshotCancelledError('Snapshot sync source was cancelled') + const config = await this.concurrentSnapshotReaderConfig() + if (config === undefined) return undefined + if (this.retainedReadSnapshotsStopped) + throw new WERR_INVALID_OPERATION('Snapshot sync is unavailable after destruction begins') + const remaining = Math.floor( + Math.min(deadline.expiresAt - Date.now(), deadline.lifetimeMs - (performance.now() - deadline.startedAt)) + ) + if (remaining < 1) throw new SnapshotResourceLimitError('Snapshot sync source expired during connection setup') + const reader = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions(this.chain), + snapshotSync: false, + telemetry: this.snapshotSyncTelemetry, + knex: createKnex(config) + }) + this.snapshotSyncSource = reader + try { + const view = await openSource(reader, identityKey, { ...options, lifetimeMs: remaining }) + if (this.retainedReadSnapshotsStopped) { + await view.close() + throw new WERR_INVALID_OPERATION('Snapshot sync is unavailable after destruction begins') + } + const closed = view.closed.finally(() => this.releaseConcurrentSource(reader)) + void closed.catch(() => undefined) + return { + ...view, + get isOpen() { + return view.isOpen + }, + closed, + close: async () => { + await view.close().catch(() => undefined) + await closed + } + } + } catch (error) { + await this.releaseConcurrentSource(reader) + throw error + } + } + + private async concurrentSnapshotReaderConfig(): Promise { + const config = this.knex.client.config as Knex.Config + const connection = config.connection + // A function or external pool cannot promise an independent connection. + // In-memory SQLite cannot share a coherent WAL view with a separate pool. + if ( + connection === null || + typeof connection !== 'object' || + ('connectionPool' in config && config.connectionPool != null) + ) + return undefined + if (this.databaseSystem() === 'sqlite') { + const filename = (connection as Knex.Sqlite3ConnectionConfig).filename + if ( + typeof filename !== 'string' || + filename.length === 0 || + filename === ':memory:' || + filename.startsWith('file:') + ) + return undefined + const modes: Array<{ journal_mode: string }> = await this.knex.raw('PRAGMA journal_mode') + if (modes[0]?.journal_mode.toLowerCase() !== 'wal') return undefined + } else if ( + this.databaseSystem() !== 'mysql' || + !('database' in connection) || + typeof connection.database !== 'string' + ) + return undefined + return { + ...config, + // Preserve hidden credentials and accessors without making them log-visible. + connection: Object.create(Object.getPrototypeOf(connection), Object.getOwnPropertyDescriptors(connection)), + pool: { ...config.pool, min: 0, max: 1 } + } + } + + private async releaseConcurrentSource(reader: StorageKnex): Promise { + try { + await reader.destroy() + } catch (error) { + // Failed physical cleanup must never reopen capacity for another view. + this.retainedReadSnapshotsStopped = true + throw error + } + if (this.snapshotSyncSource === reader) this.snapshotSyncSource = undefined + this.snapshotSyncBusy = false + } + protected override supportsNoSendExpiryPersistence(): boolean { return true } @@ -1101,6 +1616,27 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide .update(this.validatePartialForUpdate(update)) } + override async compareAndSetProvenTxProof( + expected: TableProvenTx, + replacement: TableProvenTx, + trx?: TrxToken + ): Promise { + await this.verifyReadyForDatabaseAccess(trx) + const update = recoveredProofUpdate(expected, replacement) + const where = { + provenTxId: expected.provenTxId, + txid: expected.txid, + height: expected.height, + index: expected.index, + merkleRoot: expected.merkleRoot, + blockHash: expected.blockHash, + rawTx: Buffer.from(expected.rawTx), + merklePath: Buffer.from(expected.merklePath) + } + const count = await this.toDb(trx)('proven_txs').where(where).update(this.validatePartialForUpdate(update)) + return count === 1 + } + override async updateSyncState(id: number, update: Partial, trx?: TrxToken): Promise { await this.verifyReadyForDatabaseAccess(trx) return await this.toDb(trx)('sync_states') @@ -1500,6 +2036,15 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide override async findProvenTxs(args: FindProvenTxsArgs): Promise { const q = this.findProvenTxsQuery(args) + // Writable exact-proof lookups are read/modify/write authority checks. + // Snapshot views use consistent reads; MySQL rejects FOR UPDATE in READ ONLY. + if ( + args.trx != null && + !readSnapshotTransactions.has(args.trx) && + this.dbtype === 'MySQL' && + (args.partial.txid != null || args.partial.provenTxId != null) + ) + q.forUpdate() const r = await q return this.validateEntities(r) } @@ -1515,6 +2060,7 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide // Serialize sync checkpoint reads with the page transaction on MySQL and Postgres too. if ( args.trx != null && + !readSnapshotTransactions.has(args.trx) && (this.dbtype === 'MySQL' || this.dbtype === 'Postgres') && args.partial.userId != null && (args.partial.syncStateId != null || args.partial.storageIdentityKey != null) @@ -1646,13 +2192,32 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide } override async destroy(): Promise { - await this.stopPreparedBeefTasks() - this.knex.off('query', this.onQuery) - this.knex.off('query-response', this.onQueryResponse) - this.knex.off('query-error', this.onQueryError) - for (const span of this.querySpans.values()) span.end({ status: 'cancelled' }) - this.querySpans.clear() - await this.knex?.destroy() + try { + await this.stopRetainedReadSnapshots() + } catch (error) { + // Preserve an ordinary read failure for its read consumer. A typed + // cleanup failure remains observable after the pool destruction below. + if ( + (this.guardedSnapshotFailure === undefined || this.guardedSnapshotFailure.error !== error) && + this.snapshotJournalCaptureFailure !== error && + this.snapshotJournalMaintenanceFailure !== error + ) + throw error + } finally { + await this.stopPreparedBeefTasks() + this.knex.off('query', this.onQuery) + this.knex.off('query-response', this.onQueryResponse) + this.knex.off('query-error', this.onQueryError) + for (const span of this.querySpans.values()) span.end({ status: 'cancelled' }) + this.querySpans.clear() + await this.knex?.destroy() + } + // The lifetime may have settled and cleared its slot before destroy runs. + // A settled pool alone cannot prove that a failed native close succeeded. + const failure = this.guardedSnapshotFailure?.error + if (failure instanceof SnapshotArchiveSourceCleanupError) throw failure + if (this.snapshotJournalCaptureFailure !== undefined) throw this.snapshotJournalCaptureFailure + if (this.snapshotJournalMaintenanceFailure !== undefined) throw this.snapshotJournalMaintenanceFailure } override async migrate(storageName: string, storageIdentityKey: string): Promise { @@ -1689,6 +2254,7 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide override async dropAllData(): Promise { // Only using migrations to migrate down, don't need valid properties for settings table. const migrationSource = new KnexMigrations('test', '', '', 1024) + migrationSource.migrations[SNAPSHOT_SQLITE_INDEX_MIGRATION].down = dropGenerationForDataDeletion // Check if this is a SQLite database by looking at the Knex client config const clientName = (this.knex.client as { config?: { client?: string } }).config?.client ?? '' @@ -1711,6 +2277,9 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide } try { + // An interrupted, unpublished generation has no journal entry to roll back. + // Explicit full deletion still removes its owned auxiliary schema first. + await dropGenerationForDataDeletion(this.knex) for (let i = 0; i < count; i++) { const version = await this.knex.migrate.currentVersion(config) if (version === 'none') return @@ -2254,7 +2823,7 @@ export class StorageKnex extends StorageProvider implements WalletStorageProvide void this.select(1).from('action_batch_outputs as abo').whereRaw('?? = ??', ['abo.outputId', 'o.outputId']) }) .whereIn('t.status', status) - .select('o.*') + .select('o.*') .forUpdate() let output: TableOutput | undefined diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts index 400ac4fc3..dc0a5feb9 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageProvider.ts @@ -1,3 +1,9 @@ +import type { SnapshotSyncStorage } from './snapshot/SnapshotSync' +import type { WalletReadSnapshot, WalletReadSnapshotOptions } from './snapshot/WalletReadSnapshot' +import type { RetainedReadSnapshot, RetainedReadSnapshotOptions } from './snapshot/RetainedReadSnapshot' +import { runInSeries } from '../utility/runInSeries' +import { findProofRecords, mapProofWork } from './methods/proofWork' +import { snapshotSyncPage } from './sync/snapshotSyncPage' import { type ValidCreateActionArgs, type ValidListActionsArgs, @@ -97,6 +103,7 @@ import { classifyOutputUtxo, requireConclusiveUtxo } from '../services/classifyO import { processNoSendExpiryLifecycle } from './methods/noSendExpiryLifecycle' import { canonicalizeSyncProofIdentifiers, + markSyncProofReconciled, markSyncProofInsertOnly, sameSyncProof, validateSyncProof @@ -425,6 +432,11 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal async getCapabilities(): Promise { return { + ...(this.supportsStorageAccessScheduling() + ? { + storageAccess: { version: 1 as const, concurrentReads: true, atomicSyncPages: true } + } + : {}), ...(this.supportsNoSendExpiryPersistence() ? { brc177NoSendExpiry: { version: 1 as const } } : {}), ...(this.supportsActionBatchPersistence() ? getActionBatchCapabilities(this.actionBatchMaxReservedOutputs, true) @@ -502,6 +514,52 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal return false } + /** Custom providers retain serialized manager access until explicitly supported. */ + protected supportsStorageAccessScheduling(): boolean { + return false + } + + /** Local provider contract only; this does not advertise a remote export RPC. */ + supportsReadSnapshot(): boolean { + return false + } + + /** + * Capture related rows from one database read view. The callback must use + * this token for every query and must not perform network or file I/O. + * Custom providers must implement their actual isolation guarantee rather + * than inheriting an ordinary transaction with unknown isolation. + */ + readSnapshot(_read: (trx: TrxToken) => Promise): Promise { + return Promise.reject( + new WERR_NOT_IMPLEMENTED('Coherent wallet source snapshots are not supported by this provider') + ) + } + + /** Local transaction retention only; no remote/profile export capability is implied. */ + supportsRetainedReadSnapshot(): boolean { + return false + } + + /** Older and auto-closing transaction providers must explicitly refuse retention. */ + openReadSnapshot(_options?: RetainedReadSnapshotOptions): Promise { + return Promise.reject(new WERR_NOT_IMPLEMENTED('Retained read snapshots are not supported by this provider')) + } + + /** Local, profile-bound packed keyset pages; never implied by ordinary sync support. */ + supportsWalletReadSnapshot(): boolean { + return false + } + + openWalletReadSnapshot(_identityKey: string, _options: WalletReadSnapshotOptions = {}): Promise { + return Promise.reject(new WERR_NOT_IMPLEMENTED('Wallet read snapshot pages are not supported by this provider')) + } + + /** Local versioned snapshot merge capability; older providers keep serialized sync. */ + getSnapshotSync(): SnapshotSyncStorage | undefined { + return undefined + } + protected supportsActionBatchPersistence(): boolean { return false } @@ -1196,6 +1254,11 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal return await processAction(this, auth, args) } + /** Custom providers may opt into atomic canonical-proof repair; default is no mutation. */ + compareAndSetProvenTxProof(_expected: TableProvenTx, _replacement: TableProvenTx, _trx?: TrxToken): Promise { + return Promise.resolve(false) + } + async attemptToPostReqsToNetwork( reqs: EntityProvenTxReq[], trx?: TrxToken, @@ -1389,7 +1452,8 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal }) } - async processSyncChunk(args: RequestSyncChunkArgs, chunk: SyncChunk): Promise { + protected async prepareSyncProofs(chunk: SyncChunk): Promise> { + const expected = new Map() // Canonicalize before text-key lookup in every sync mode. Direct // backup/conflict sync retains its established trust for new proof rows, // but may replace an existing global proof only after active-chain @@ -1397,23 +1461,64 @@ export abstract class StorageProvider extends StorageReaderWriter implements Wal const incomingProofs = chunk.provenTxs ?? [] for (const proof of incomingProofs) canonicalizeSyncProofIdentifiers(proof) if (incomingProofs.length > 0) { - const existingProofs = await this.findProvenTxs({ - partial: {}, - txids: [...new Set(incomingProofs.map(proof => proof.txid))] - }) + const existingProofs = await findProofRecords( + this, + incomingProofs.map(proof => proof.txid) + ) const existingByTxid = new Map(existingProofs.map(proof => [proof.txid.toLowerCase(), proof])) - for (const proof of incomingProofs) { + await mapProofWork(incomingProofs, async proof => { const existing = existingByTxid.get(proof.txid) if (existing == null) markSyncProofInsertOnly(proof) - else if (!sameSyncProof(existing, proof)) await validateSyncProof(this, proof) - } + else if (!sameSyncProof(existing, proof)) { + await validateSyncProof(this, proof) + markSyncProofReconciled(proof) + expected.set(proof.txid, existing) + } + }) } + return expected + } + + /** Local-only preparation: detach and validate proof I/O before taking manager write ownership. */ + async prepareSyncChunk(args: RequestSyncChunkArgs, chunk: SyncChunk): Promise<() => Promise> { + const snapshot = snapshotSyncPage(args, chunk) + const expected = await this.prepareSyncProofs(snapshot.chunk) + let consumed = false + return async () => { + if (consumed) + throw new WERR_INVALID_OPERATION('Prepared sync page already consumed; resume from its durable checkpoint.') + consumed = true + return await this.commitSyncChunk(snapshot.args, snapshot.chunk, expected) + } + } + + async processSyncChunk(args: RequestSyncChunkArgs, chunk: SyncChunk): Promise { + return await this.commitSyncChunk(args, chunk, await this.prepareSyncProofs(chunk)) + } + + private async commitSyncChunk( + args: RequestSyncChunkArgs, + chunk: SyncChunk, + expected: Map + ): Promise { // A sync page may contain hundreds of related entities. Keep their lookups, // inserts, ID remapping, and sync-state checkpoint in one transaction. This // avoids a transaction startup/commit for every record (especially costly // in IndexedDB) and makes the page checkpoint atomic with its data changes. return await this.transaction(async trx => { + // Serial sorted reads retain deterministic lock ordering on the shared transaction. + await runInSeries( + [...expected.values()].sort((left, right) => left.txid.localeCompare(right.txid)), + async previous => { + const current = verifyOneOrNone(await this.findProvenTxs({ partial: { txid: previous.txid }, trx })) + if (current == null || !sameSyncProof(current, previous)) { + throw new WERR_INVALID_OPERATION( + 'Proof changed during sync preparation; resume from the durable checkpoint.' + ) + } + } + ) const user = verifyTruthy( verifyOneOrNone(await this.findUsers({ partial: { identityKey: args.identityKey }, trx })) ) diff --git a/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts b/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts index b77ff11e2..8c689da96 100644 --- a/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts +++ b/packages/wallet/wallet-toolbox/src/storage/StorageReader.ts @@ -116,8 +116,8 @@ export abstract class StorageReader implements sdk.WalletStorageSyncReader { return undefined } - async findUserByIdentityKey(key: string): Promise { - return verifyOneOrNone(await this.findUsers({ partial: { identityKey: key } })) + async findUserByIdentityKey(key: string, trx?: sdk.TrxToken): Promise { + return verifyOneOrNone(await this.findUsers({ partial: { identityKey: key }, trx })) } async getSyncChunk(args: sdk.RequestSyncChunkArgs): Promise { diff --git a/packages/wallet/wallet-toolbox/src/storage/WalletStorageManager.ts b/packages/wallet/wallet-toolbox/src/storage/WalletStorageManager.ts index c603999a4..67935273e 100644 --- a/packages/wallet/wallet-toolbox/src/storage/WalletStorageManager.ts +++ b/packages/wallet/wallet-toolbox/src/storage/WalletStorageManager.ts @@ -1,3 +1,10 @@ +import { SnapshotResourceLimitError } from './snapshot/SnapshotResourceLimitError' +import { SnapshotCancelledError } from './snapshot/SnapshotCancelledError' +import { copySnapshotCursor } from './snapshot/SnapshotCursor' +import { runSnapshotSyncSession } from './snapshot/runSnapshotSyncSession' +import type { SnapshotSyncStorage } from './snapshot/SnapshotSync' +import type { WalletReadSnapshot } from './snapshot/WalletReadSnapshot' +import { runInSeries } from '../utility/runInSeries' import { type ValidCreateActionArgs, type ValidListActionsArgs, @@ -9,17 +16,23 @@ import { validateRelinquishOutputArgs } from '@bsv/sdk/wallet/validationHelpers' import { SyncPageBudget } from './sync/SyncPageBudget' +import { StorageAccessQueue } from './sync/StorageAccessQueue' +import { + runPullSession, + type SyncSessionOptions, + type SyncSessionProgress, + type SyncSessionResult +} from './sync/syncSession' import { validateSyncCheckpoint } from './sync/syncCheckpoint' +import { assertSyncNetwork, assertSyncProgress, throwSyncResultError } from './sync/syncFailure' import { AbortActionArgs, AbortActionResult, Beef, - ChainTracker, InternalizeActionArgs, ListActionsResult, ListCertificatesResult, ListOutputsResult, - MerklePath, RelinquishCertificateArgs, RelinquishOutputArgs } from '@bsv/sdk' @@ -44,7 +57,9 @@ import { TableUser } from '../storage/schema/tables' import { StorageProvider } from './StorageProvider' -import { getCanonicalMerklePath } from '../services/getCanonicalMerklePath' +import { refreshSyncProof } from './methods/refreshSyncProof' +import { recoveredProofUpdate, sameSyncProof } from './methods/validateSyncProof' +import { mapProofWork } from './methods/proofWork' interface PreparedBeefInvalidationExtension { invalidatePreparedBeefs: (trx?: sdk.TrxToken) => Promise @@ -62,6 +77,7 @@ class ManagedStorage { isAvailable: boolean isStorageProvider: boolean settings?: TableSettings + access: sdk.StorageCapabilities['storageAccess'] user?: TableUser constructor(public storage: sdk.WalletStorageProvider) { @@ -114,6 +130,9 @@ export class WalletStorageManager implements sdk.WalletStorage { * Configured services if any. If valid, shared with stores (which may ignore it). */ _services?: sdk.WalletServices + private availability?: Promise + private generation = 0 + private readonly accessQueue = new StorageAccessQueue() /** * Creates a new WalletStorageManager with the given identityKey and optional active and backup storage providers. @@ -172,12 +191,30 @@ export class WalletStorageManager implements sdk.WalletStorage { */ private async ensureStoreAvailable(store: ManagedStorage): Promise { if (store.isAvailable && store.settings != null && store.user != null) return - store.settings = await store.storage.makeAvailable() + store.settings ??= await store.storage.makeAvailable() const r = await store.storage.findOrInsertUser(this._authId.identityKey) store.user = r.user + // A failed capability lookup changes only scheduling: old or unavailable + // capability endpoints keep the compatible serialized path. + try { + const access = (await store.storage.getCapabilities?.())?.storageAccess + store.access = access?.version === 1 ? access : undefined + } catch { + store.access = undefined + } store.isAvailable = true } + private async preflightManagedNetworks(peer?: TableSettings): Promise { + let reference = peer + // The first available store establishes the network for each later store. + await runInSeries(this._stores, async store => { + store.settings ??= await store.storage.makeAvailable() + if (reference != null) assertSyncNetwork(reference, store.settings) + reference ??= store.settings + }) + } + private selectActiveFromStore(store: ManagedStorage, backups: ManagedStorage[]): void { if (this._active == null) { // _stores[0] becomes the default active store. @@ -196,6 +233,17 @@ export class WalletStorageManager implements sdk.WalletStorage { } async makeAvailable(): Promise { + if (this._isAvailable) return this.getActiveSettings() + this.availability ??= this.initializeAvailable() + const pending = this.availability + try { + return await pending + } finally { + if (this.availability === pending) this.availability = undefined + } + } + + private async initializeAvailable(): Promise { if (this._isAvailable) return (this._active as ManagedStorage).settings as TableSettings this._active = undefined @@ -207,6 +255,8 @@ export class WalletStorageManager implements sdk.WalletStorage { } const backups: ManagedStorage[] = [] + // Read all network settings before registering users on any managed store. + await this.preflightManagedNetworks() for (const store of this._stores) { await this.ensureStoreAvailable(store) this.selectActiveFromStore(store, backups) @@ -220,6 +270,7 @@ export class WalletStorageManager implements sdk.WalletStorage { } this._isAvailable = true + this.generation++ this._authId.userId = (this._active as unknown as ManagedStorage).user?.userId this._authId.isActive = this.isActiveEnabled @@ -280,134 +331,78 @@ export class WalletStorageManager implements sdk.WalletStorage { return this._stores.map(b => (b.settings as TableSettings).storageIdentityKey) } - private readonly readerLocks: Array<(value: void | PromiseLike) => void> = [] - private readonly writerLocks: Array<(value: void | PromiseLike) => void> = [] - private readonly syncLocks: Array<(value: void | PromiseLike) => void> = [] - private readonly spLocks: Array<(value: void | PromiseLike) => void> = [] - - private async getActiveLock(lockQueue: Array<(value: void | PromiseLike) => void>): Promise { - if (!this.isAvailable()) await this.makeAvailable() - - let resolveNewLock: () => void = () => {} - const newLock = new Promise(resolve => { - resolveNewLock = resolve - lockQueue.push(resolve) - }) - if (lockQueue.length === 1) { - resolveNewLock() + private async withAccess( + operation: (active: sdk.WalletStorageProvider) => Promise, + read = false, + background = false + ): Promise { + await this.makeAvailable() + const concurrent = read && this._active?.access?.concurrentReads === true + const release = await this.accessQueue.acquire( + concurrent ? 'read' : 'exclusive', + background ? 'background' : 'foreground' + ) + // A preceding transition can fail after updating a managed store. Reload + // its persisted selection under ownership before using the cached provider. + if (!this._isAvailable) { + try { + await this.makeAvailable() + } catch (error) { + release() + throw error + } } - await newLock - } - - private releaseActiveLock(queue: Array<(value: void | PromiseLike) => void>): void { - queue.shift() // Remove the current lock from the queue - if (queue.length > 0) { - queue[0]() + // Primary selection may have changed while this request was queued. Never + // apply the former provider's read-sharing promise to its replacement. + if (concurrent && this._active?.access?.concurrentReads !== true) { + release() + return await this.withAccess(operation, read, background) + } + try { + return await operation(this.getActive()) + } finally { + release() } } - private async getActiveForReader(): Promise { - await this.getActiveLock(this.readerLocks) - return this.getActive() - } - - private releaseActiveForReader(): void { - this.releaseActiveLock(this.readerLocks) - } - - private async getActiveForWriter(): Promise { - await this.getActiveLock(this.readerLocks) - await this.getActiveLock(this.writerLocks) - return this.getActive() - } - - private releaseActiveForWriter(): void { - this.releaseActiveLock(this.writerLocks) - this.releaseActiveLock(this.readerLocks) - } - - private async getActiveForSync(): Promise { - await this.getActiveLock(this.readerLocks) - await this.getActiveLock(this.writerLocks) - await this.getActiveLock(this.syncLocks) - return this.getActive() - } - - private releaseActiveForSync(): void { - this.releaseActiveLock(this.syncLocks) - this.releaseActiveLock(this.writerLocks) - this.releaseActiveLock(this.readerLocks) + runAsWriter(writer: (active: sdk.WalletStorageWriter) => Promise): Promise { + return this.withAccess(writer) } - private async getActiveForStorageProvider(): Promise { - await this.getActiveLock(this.readerLocks) - await this.getActiveLock(this.writerLocks) - await this.getActiveLock(this.syncLocks) - await this.getActiveLock(this.spLocks) - - const active = this.getActive() - // We can finally confirm that active storage is still able to support `StorageProvider` - if (!active.isStorageProvider()) { - throw new WERR_INVALID_OPERATION('Active "WalletStorageProvider" does not support "StorageProvider" interface.') - } - // Allow the sync to proceed on the active store. - return active as unknown as StorageProvider - } - - private releaseActiveForStorageProvider(): void { - this.releaseActiveLock(this.spLocks) - this.releaseActiveLock(this.syncLocks) - this.releaseActiveLock(this.writerLocks) - this.releaseActiveLock(this.readerLocks) + /** Keep foreground writer authorization inside the acquired ownership boundary. */ + private runAsAuthorizedWriter( + operation: (writer: sdk.WalletStorageWriter, auth: sdk.AuthId) => Promise + ): Promise { + return this.runAsWriter(async writer => operation(writer, await this.getAuth(true))) } - async runAsWriter(writer: (active: sdk.WalletStorageWriter) => Promise): Promise { - try { - const active = await this.getActiveForWriter() - const r = await writer(active) - return r - } finally { - this.releaseActiveForWriter() - } + runAsReader(reader: (active: sdk.WalletStorageReader) => Promise): Promise { + return this.withAccess(reader, true) } - async runAsReader(reader: (active: sdk.WalletStorageReader) => Promise): Promise { - try { - const active = await this.getActiveForReader() - const r = await reader(active) - return r - } finally { - this.releaseActiveForReader() - } + /** Preserve the legacy reader contract: obtain caller auth before joining the ownership queue. */ + private async runAsAuthorizedReader( + operation: (reader: sdk.WalletStorageReader, auth: sdk.AuthId) => Promise + ): Promise { + const auth = await this.getAuth() + return this.runAsReader(reader => operation(reader, auth)) } - /** - * - * @param sync the function to run with sync access lock - * @param activeSync from chained sync functions, active storage already held under sync access lock. - * @returns - */ + /** Borrowed activeSync is the legacy explicit reentrancy contract for an already-held exclusive operation. */ async runAsSync( sync: (active: sdk.WalletStorageSync) => Promise, activeSync?: sdk.WalletStorageSync ): Promise { - try { - const active = activeSync ?? (await this.getActiveForSync()) - const r = await sync(active) - return r - } finally { - if (activeSync == null) this.releaseActiveForSync() - } + return activeSync == null ? await this.withAccess(sync) : await sync(activeSync) } - async runAsStorageProvider(sync: (active: StorageProvider) => Promise): Promise { - try { - const active = await this.getActiveForStorageProvider() - const r = await sync(active) - return r - } finally { - this.releaseActiveForStorageProvider() - } + runAsStorageProvider(sync: (active: StorageProvider) => Promise): Promise { + return this.withAccess(active => { + if (!active.isStorageProvider()) { + throw new WERR_INVALID_OPERATION('Active "WalletStorageProvider" does not support "StorageProvider" interface.') + } + return sync(active as unknown as StorageProvider) + }) } /** @@ -442,11 +437,18 @@ export class WalletStorageManager implements sdk.WalletStorage { } async addWalletStorageProvider(provider: sdk.WalletStorageProvider): Promise { - await provider.makeAvailable() - if (this._services != null) provider.setServices(this._services) - this._stores.push(new ManagedStorage(provider)) - this._isAvailable = false - await this.makeAvailable() + const settings = await provider.makeAvailable() + const add = async (): Promise => { + await this.preflightManagedNetworks(settings) + if (this._services != null) provider.setServices(this._services) + const store = new ManagedStorage(provider) + store.settings = settings + this._stores.push(store) + this._isAvailable = false + await this.makeAvailable() + } + if (this._stores.length === 0) await add() + else await this.withAccess(add) } setServices(v: sdk.WalletServices): void { @@ -463,15 +465,14 @@ export class WalletStorageManager implements sdk.WalletStorage { return this.getActive().getSettings() } - async migrate(storageName: string, storageIdentityKey: string): Promise { - return await this.runAsWriter(async writer => { - return await writer.migrate(storageName, storageIdentityKey) - }) + migrate(storageName: string, storageIdentityKey: string): Promise { + return this.runAsWriter(writer => writer.migrate(storageName, storageIdentityKey)) } async destroy(): Promise { if (this._stores.length < 1) return return await this.runAsWriter(async _writer => { + this.generation++ for (const store of this._stores) await store.storage.destroy() }) } @@ -493,52 +494,34 @@ export class WalletStorageManager implements sdk.WalletStorage { async abortAction(args: AbortActionArgs): Promise { validateAbortActionArgs(args) - return await this.runAsWriter(async writer => { - const auth = await this.getAuth(true) - return await writer.abortAction(auth, args) - }) + return await this.runAsAuthorizedWriter((writer, auth) => writer.abortAction(auth, args)) } - async createAction(vargs: ValidCreateActionArgs): Promise { - return await this.runAsWriter(async writer => { - const auth = await this.getAuth(true) - return await writer.createAction(auth, vargs) - }) + createAction(vargs: ValidCreateActionArgs): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.createAction(auth, vargs)) } async internalizeAction(args: InternalizeActionArgs): Promise { validateInternalizeActionArgs(args) - return await this.runAsWriter(async writer => { - const auth = await this.getAuth(true) - return await writer.internalizeAction(auth, args) - }) + return await this.runAsAuthorizedWriter((writer, auth) => writer.internalizeAction(auth, args)) } async relinquishCertificate(args: RelinquishCertificateArgs): Promise { validateRelinquishCertificateArgs(args) - return await this.runAsWriter(async writer => { - const auth = await this.getAuth(true) - return await writer.relinquishCertificate(auth, args) - }) + return await this.runAsAuthorizedWriter((writer, auth) => writer.relinquishCertificate(auth, args)) } async relinquishOutput(args: RelinquishOutputArgs): Promise { validateRelinquishOutputArgs(args) - return await this.runAsWriter(async writer => { - const auth = await this.getAuth(true) - return await writer.relinquishOutput(auth, args) - }) + return await this.runAsAuthorizedWriter((writer, auth) => writer.relinquishOutput(auth, args)) } - async processAction(args: sdk.StorageProcessActionArgs): Promise { - return await this.runAsWriter(async writer => { - const auth = await this.getAuth(true) - return await writer.processAction(auth, args) - }) + processAction(args: sdk.StorageProcessActionArgs): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.processAction(auth, args)) } - async prepareNoSendExpiry(args: ValidCreateActionArgs): Promise { - return await this.runAsWriter(async writer => { + prepareNoSendExpiry(args: ValidCreateActionArgs): Promise { + return this.runAsWriter(async writer => { if (writer.prepareNoSendExpiry == null) { throw new WERR_INVALID_OPERATION('Active storage does not support BRC-177 noSend expiry') } @@ -546,10 +529,8 @@ export class WalletStorageManager implements sdk.WalletStorage { }) } - async activateNoSendExpiry( - args: sdk.StorageActivateNoSendExpiryArgs - ): Promise { - return await this.runAsWriter(async writer => { + activateNoSendExpiry(args: sdk.StorageActivateNoSendExpiryArgs): Promise { + return this.runAsWriter(async writer => { if (writer.activateNoSendExpiry == null) { throw new WERR_INVALID_OPERATION('Active storage does not support BRC-177 noSend expiry') } @@ -566,24 +547,24 @@ export class WalletStorageManager implements sdk.WalletStorage { }) } - async getCapabilities(): Promise { - return await this.runAsReader(async () => await this.getActive().getCapabilities()) + getCapabilities(): Promise { + return this.runAsReader(() => this.getActive().getCapabilities()) } - async beginActionBatch(args: sdk.BeginActionBatchArgs): Promise { - return await this.runAsWriter(async writer => await writer.beginActionBatch(await this.getAuth(true), args)) + beginActionBatch(args: sdk.BeginActionBatchArgs): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.beginActionBatch(auth, args)) } - async extendActionBatch(args: sdk.ExtendActionBatchArgs): Promise { - return await this.runAsWriter(async writer => await writer.extendActionBatch(await this.getAuth(true), args)) + extendActionBatch(args: sdk.ExtendActionBatchArgs): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.extendActionBatch(auth, args)) } - async renewActionBatch(batchId: string): Promise { - return await this.runAsWriter(async writer => await writer.renewActionBatch(await this.getAuth(true), batchId)) + renewActionBatch(batchId: string): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.renewActionBatch(auth, batchId)) } - async resumeActionBatch(args: sdk.ResumeActionBatchArgs): Promise { - return await this.runAsWriter(async writer => { + resumeActionBatch(args: sdk.ResumeActionBatchArgs): Promise { + return this.runAsWriter(async writer => { if (writer.resumeActionBatch == null) { throw new WERR_NOT_IMPLEMENTED('action batch resume is not available') } @@ -591,18 +572,16 @@ export class WalletStorageManager implements sdk.WalletStorage { }) } - async prepareActionBatchCommit(manifest: sdk.ActionBatchManifest): Promise { - return await this.runAsWriter( - async writer => await writer.prepareActionBatchCommit(await this.getAuth(true), manifest) - ) + prepareActionBatchCommit(manifest: sdk.ActionBatchManifest): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.prepareActionBatchCommit(auth, manifest)) } - async putActionBatchBlob(args: sdk.PutActionBatchBlobArgs): Promise { - return await this.runAsWriter(async writer => await writer.putActionBatchBlob(await this.getAuth(true), args)) + putActionBatchBlob(args: sdk.PutActionBatchBlobArgs): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.putActionBatchBlob(auth, args)) } - async putActionBatchPack(args: sdk.PutActionBatchPackArgs): Promise { - return await this.runAsWriter(async writer => { + putActionBatchPack(args: sdk.PutActionBatchPackArgs): Promise { + return this.runAsWriter(async writer => { if (writer.putActionBatchPack == null) { throw new WERR_NOT_IMPLEMENTED('packed action batch uploads are not available') } @@ -610,12 +589,12 @@ export class WalletStorageManager implements sdk.WalletStorage { }) } - async commitActionBatch(manifest: sdk.ActionBatchManifest): Promise { - return await this.runAsWriter(async writer => await writer.commitActionBatch(await this.getAuth(true), manifest)) + commitActionBatch(manifest: sdk.ActionBatchManifest): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.commitActionBatch(auth, manifest)) } - async commitActionBatchByDigest(args: sdk.CommitActionBatchByDigestArgs): Promise { - return await this.runAsWriter(async writer => { + commitActionBatchByDigest(args: sdk.CommitActionBatchByDigestArgs): Promise { + return this.runAsWriter(async writer => { if (writer.commitActionBatchByDigest == null) { throw new WERR_NOT_IMPLEMENTED('digest-only action batch commit is not available') } @@ -623,63 +602,40 @@ export class WalletStorageManager implements sdk.WalletStorage { }) } - async abortActionBatch(batchId: string): Promise { - return await this.runAsWriter(async writer => await writer.abortActionBatch(await this.getAuth(true), batchId)) + abortActionBatch(batchId: string): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.abortActionBatch(auth, batchId)) } - async insertCertificate(certificate: TableCertificate): Promise { - return await this.runAsWriter(async writer => { - const auth = await this.getAuth(true) - return await writer.insertCertificateAuth(auth, certificate) - }) + insertCertificate(certificate: TableCertificate): Promise { + return this.runAsAuthorizedWriter((writer, auth) => writer.insertCertificateAuth(auth, certificate)) } - async listActions(vargs: ValidListActionsArgs): Promise { - const auth = await this.getAuth() - return await this.runAsReader(async reader => { - return await reader.listActions(auth, vargs) - }) + listActions(vargs: ValidListActionsArgs): Promise { + return this.runAsAuthorizedReader((reader, auth) => reader.listActions(auth, vargs)) } - async listCertificates(args: ValidListCertificatesArgs): Promise { - const auth = await this.getAuth() - return await this.runAsReader(async reader => { - return await reader.listCertificates(auth, args) - }) + listCertificates(args: ValidListCertificatesArgs): Promise { + return this.runAsAuthorizedReader((reader, auth) => reader.listCertificates(auth, args)) } - async listOutputs(vargs: ValidListOutputsArgs): Promise { - const auth = await this.getAuth() - return await this.runAsReader(async reader => { - return await reader.listOutputs(auth, vargs) - }) + listOutputs(vargs: ValidListOutputsArgs): Promise { + return this.runAsAuthorizedReader((reader, auth) => reader.listOutputs(auth, vargs)) } - async findCertificates(args: sdk.FindCertificatesArgs): Promise { - const auth = await this.getAuth() - return await this.runAsReader(async reader => { - return await reader.findCertificatesAuth(auth, args) - }) + findCertificates(args: sdk.FindCertificatesArgs): Promise { + return this.runAsAuthorizedReader((reader, auth) => reader.findCertificatesAuth(auth, args)) } - async findOutputBaskets(args: sdk.FindOutputBasketsArgs): Promise { - const auth = await this.getAuth() - return await this.runAsReader(async reader => { - return await reader.findOutputBasketsAuth(auth, args) - }) + findOutputBaskets(args: sdk.FindOutputBasketsArgs): Promise { + return this.runAsAuthorizedReader((reader, auth) => reader.findOutputBasketsAuth(auth, args)) } - async findOutputs(args: sdk.FindOutputsArgs): Promise { - const auth = await this.getAuth() - return await this.runAsReader(async reader => { - return await reader.findOutputsAuth(auth, args) - }) + findOutputs(args: sdk.FindOutputsArgs): Promise { + return this.runAsAuthorizedReader((reader, auth) => reader.findOutputsAuth(auth, args)) } - async findProvenTxReqs(args: sdk.FindProvenTxReqsArgs): Promise { - return await this.runAsReader(async reader => { - return await reader.findProvenTxReqs(args) - }) + findProvenTxReqs(args: sdk.FindProvenTxReqsArgs): Promise { + return this.runAsReader(reader => reader.findProvenTxReqs(args)) } /** @@ -691,165 +647,124 @@ export class WalletStorageManager implements sdk.WalletStorage { * @returns */ async reproveHeader(deactivatedHash: string): Promise { - const r: sdk.ReproveHeaderResult = { log: '', updated: [], unchanged: [], unavailable: [] } - - // Lookup all the proven_txs records matching the deactivated headers - let ptxs: TableProvenTx[] = [] - await this.runAsStorageProvider(async sp => { - ptxs = await sp.findProvenTxs({ partial: { blockHash: deactivatedHash } }) - }) - - r.log += ` block ${deactivatedHash} orphaned with ${ptxs.length} impacted transactions\n` - - for (const ptx of ptxs) { - // Loop over proven_txs records matching the deactivated header - const rp = await this.reproveProven(ptx, true) - - r.log += rp.log - if (rp.unavailable) r.unavailable.push(ptx) - if (rp.unchanged) r.unchanged.push(ptx) - if (rp.updated != null) r.updated.push({ was: ptx, update: rp.updated.update, logUpdate: rp.updated.logUpdate }) - } - - // Invalidate as soon as storage contains proof data for the deactivated - // header, even when a replacement proof is not available yet. A prepared - // artifact carrying the old proof must not remain readable during retries. - if (ptxs.length > 0) { - await this.runAsStorageProvider(async sp => { - await sp.transaction(async trx => { - for (const u of r.updated) { - await sp.updateProvenTx(u.was.provenTxId, u.update, trx) - r.log += ` txid ${u.was.txid} proof data updated\n` + u.logUpdate - } - await invalidatePreparedBeefs(sp, trx) - }) - }) - } - - return r + return await this.reproveMatching({ blockHash: deactivatedHash }, `block ${deactivatedHash} orphaned`) } - /** - * For all proven_txs records at the given height currently tied to the given stale merkleRoot, - * attempt to reprove them against the current chain and update proof data if new valid proofs are found. - * - * This is intended for backup auditing of recent heights after the primary reorg event path has run. - */ + /** Audit a stale root against the same canonical evidence as reorg recovery. */ async reproveHeightMerkleRoot(height: number, staleMerkleRoot: string): Promise { - const r: sdk.ReproveHeaderResult = { log: '', updated: [], unchanged: [], unavailable: [] } - - let ptxs: TableProvenTx[] = [] - await this.runAsStorageProvider(async sp => { - ptxs = await sp.findProvenTxs({ partial: { height, merkleRoot: staleMerkleRoot } }) - }) - - r.log += ` height ${height} stale merkleRoot ${staleMerkleRoot} with ${ptxs.length} impacted transactions\n` - - for (const ptx of ptxs) { - const rp = await this.reproveProven(ptx, true) + return await this.reproveMatching( + { height, merkleRoot: staleMerkleRoot }, + `height ${height} stale merkleRoot ${staleMerkleRoot}` + ) + } - r.log += rp.log - if (rp.unavailable) r.unavailable.push(ptx) - if (rp.unchanged) r.unchanged.push(ptx) - if (rp.updated != null) r.updated.push({ was: ptx, update: rp.updated.update, logUpdate: rp.updated.logUpdate }) + private assertProofDestination(storage: StorageProvider, generation: number): void { + if (this.getActive() !== storage || this.generation !== generation) { + throw new WERR_INVALID_OPERATION( + 'Proof destination changed during recovery; retry on the selected storage provider.' + ) } + } - // A matching stale root invalidates prepared proof material whether or not - // this audit can obtain a replacement proof in the same pass. - if (ptxs.length > 0) { - await this.runAsStorageProvider(async sp => { - await sp.transaction(async trx => { - for (const u of r.updated) { - await sp.updateProvenTx(u.was.provenTxId, u.update, trx) - r.log += ` txid ${u.was.txid} proof data updated\n` + u.logUpdate - } - await invalidatePreparedBeefs(sp, trx) - }) - }) + private async prepareReproof( + storage: StorageProvider, + ptx: TableProvenTx + ): Promise<{ + result: sdk.ReproveProvenResult + replacement?: TableProvenTx + }> { + const result: sdk.ReproveProvenResult = { log: '', updated: undefined, unchanged: false, unavailable: false } + try { + const replacement = await refreshSyncProof(storage, ptx) + if (sameSyncProof(ptx, replacement)) { + result.unchanged = true + result.log = ` txid ${ptx.txid} canonical proof unchanged\n` + return { result } + } + result.updated = { + update: recoveredProofUpdate(ptx, replacement), + logUpdate: ` height ${ptx.height} -> ${replacement.height}\n` + } + return { result, replacement } + } catch { + result.unavailable = true + result.log = ` txid ${ptx.txid} canonical proof unavailable\n` + return { result } } - - return r } - /** - * Attempt to reprove the transaction against the current chain, - * If a new valid proof is found and noUpdate is not true, - * update the proven_txs record with new block and merkle proof data. - * If noUpdate is true, the update to be applied is available in the returned result. - * - * @param ptx proven_txs record to reprove - * @param noUpdate - * @returns - */ - private async evaluateNewMerkleLeaf( - ptx: TableProvenTx, - mp: MerklePath, - leaf: { offset: number }, - blockHash: string, - chaintracker: ChainTracker, - r: sdk.ReproveProvenResult, - update: Partial - ): Promise { - if (blockHash === ptx.blockHash) { - r.log += ` txid ${ptx.txid} merkle path update still based on deactivated header ${ptx.blockHash}\n` - r.unchanged = true - return - } - const merkleRoot = mp.computeRoot(ptx.txid) - const isValid = await chaintracker.isValidRootForHeight(merkleRoot, update.height as number) - const heightChange = ptx.height === update.height ? 'unchanged' : `-> ${String(update.height)}` - const logUpdate = ` height ${ptx.height} ${heightChange}\n` - r.log += ` blockHash ${ptx.blockHash} -> ${String(update.blockHash)}\n` - r.log += ` merkleRoot ${ptx.merkleRoot} -> ${String(update.merkleRoot)}\n` - r.log += ` index ${ptx.index} -> ${String(update.index)}\n` - if (isValid === true) { - r.updated = { update, logUpdate } - } else { - r.log += ` txid ${ptx.txid} chaintracker fails to confirm updated merkle path update invalid\n` + logUpdate - r.unavailable = true + private async reproveMatching(partial: Partial, label: string): Promise { + const { storage, generation, ptxs } = await this.runAsStorageProvider(async storage => ({ + storage, + generation: this.generation, + ptxs: await storage.findProvenTxs({ partial }) + })) + // Bound external work and leave foreground storage access available while + // providers fetch proofs/headers. Every replacement is validated before SQL/IDB. + const prepared = await mapProofWork(ptxs, ptx => this.prepareReproof(storage, ptx)) + const result: sdk.ReproveHeaderResult = { + log: ` ${label} with ${ptxs.length} impacted transactions\n`, + updated: [], + unchanged: [], + unavailable: [] } + await this.runAsStorageProvider(async active => { + this.assertProofDestination(storage, generation) + if (ptxs.length === 0) return + await active.transaction(async trx => { + // Keep proof writes and their result log in source order within this transaction. + await runInSeries(ptxs.entries(), async ([index, ptx]) => { + const { result: proof, replacement } = prepared[index] + result.log += proof.log + if (replacement !== undefined && proof.updated !== undefined) { + if (await active.compareAndSetProvenTxProof(ptx, replacement, trx)) { + result.updated.push({ was: ptx, ...proof.updated }) + result.log += ` txid ${ptx.txid} proof data updated\n` + proof.updated.logUpdate + } else { + result.unavailable.push(ptx) + result.log += ` txid ${ptx.txid} proof changed concurrently or provider cannot commit safely; retry\n` + } + } else if (proof.unchanged) result.unchanged.push(ptx) + else result.unavailable.push(ptx) + }) + // Even unavailable replacements invalidate material built from the + // orphaned header. Proof rows and the prepared epoch commit atomically. + await invalidatePreparedBeefs(active, trx) + }) + }) + return result } + /** Validate current-chain evidence; noUpdate returns a proposal without persisting it. */ async reproveProven(ptx: TableProvenTx, noUpdate?: boolean): Promise { - const r: sdk.ReproveProvenResult = { log: '', updated: undefined, unchanged: false, unavailable: false } - const services = this.getServices() - const chaintracker = await services.getChainTracker() - - const mpr = await getCanonicalMerklePath(services, chaintracker, ptx.txid) - if (mpr.merklePath != null && mpr.header != null) { - const mp = mpr.merklePath - const h = mpr.header - const leaf = mp.path[0].find(leaf => leaf.txid === true && leaf.hash === ptx.txid) - if (leaf != null) { - const update: Partial = { - height: mp.blockHeight, - index: leaf.offset, - merklePath: mp.toBinary(), - merkleRoot: h.merkleRoot, - blockHash: h.hash - } - await this.evaluateNewMerkleLeaf(ptx, mp, leaf, h.hash, chaintracker, r, update) - } else { - r.log += ` txid ${ptx.txid} merkle path update doesn't include txid\n` - r.unavailable = true - } - } else { - r.log += ` txid ${ptx.txid} merkle path update unavailable\n` - r.unavailable = true + // The caller retains its input while proof I/O runs outside queue ownership. + ptx = { + ...ptx, + rawTx: ptx.rawTx.slice(), + merklePath: ptx.merklePath.slice(), + created_at: new Date(ptx.created_at), + updated_at: new Date(ptx.updated_at) } - - if (r.updated != null && noUpdate !== true) { - const updatedSnapshot = r.updated - await this.runAsStorageProvider(async sp => { - await sp.transaction(async trx => { - await sp.updateProvenTx(ptx.provenTxId, updatedSnapshot.update, trx) - await invalidatePreparedBeefs(sp, trx) - r.log += ` txid ${ptx.txid} proof data updated\n` + updatedSnapshot.logUpdate - }) + const { storage, generation } = await this.runAsStorageProvider(storage => + Promise.resolve({ storage, generation: this.generation }) + ) + const { result, replacement } = await this.prepareReproof(storage, ptx) + await this.runAsStorageProvider(async active => { + this.assertProofDestination(storage, generation) + if (replacement === undefined || result.updated === undefined || noUpdate === true) return + const updated = result.updated + await active.transaction(async trx => { + if (await active.compareAndSetProvenTxProof(ptx, replacement, trx)) { + await invalidatePreparedBeefs(active, trx) + result.log += ` txid ${ptx.txid} proof data updated\n` + updated.logUpdate + } else { + result.updated = undefined + result.unavailable = true + result.log += ` txid ${ptx.txid} proof changed concurrently or provider cannot commit safely; retry\n` + } }) - } - - return r + }) + return result } private async loadSyncRequest( @@ -877,28 +792,207 @@ export class WalletStorageManager implements sdk.WalletStorage { return ss.makeRequestSyncChunkArgs(auth.identityKey, toStorageIdentityKey) } + private async runSnapshotCopy( + view: WalletReadSnapshot, + destination: SnapshotSyncStorage, + direction: 'push' | 'pull', + options: SyncSessionOptions, + generation: number, + active: sdk.WalletStorageProvider + ): Promise { + if (view.user.identityKey !== this._authId.identityKey) throw new WERR_UNAUTHORIZED() + return await runSnapshotSyncSession( + { + view, + destination, + activeStorage: direction === 'push' ? view.user.activeStorage : this.getActiveUser().activeStorage, + commit: operation => + this.withAccess( + () => { + if (generation !== this.generation || active !== this.getActive()) { + throw new WERR_INVALID_OPERATION( + 'Snapshot sync primary generation changed; resume on the selected storage' + ) + } + return operation() + }, + false, + true + ) + }, + options + ) + } + + private committedPageLog(progress: SyncSessionProgress, logger: (message: string) => string): string { + if (progress.state !== 'committed') return '' + return logger( + `chunk ${progress.pages - 1} committed: ${progress.inserts} total inserts, ${progress.updates} total updates\n` + ) + } + + private async runLegacySnapshotFallback( + reader: sdk.WalletStorageSyncReader, + writer: sdk.WalletStorageProvider, + direction: 'push' | 'pull', + options: SyncSessionOptions, + selected: { generation: number; active: sdk.WalletStorageProvider } + ): Promise { + const readerSettings = await reader.makeAvailable() + const writerSettings = await writer.makeAvailable() + assertSyncNetwork(readerSettings, writerSettings) + return await this.runAsSync(() => { + if (selected.generation !== this.generation || selected.active !== this.getActive()) { + throw new WERR_INVALID_OPERATION('Snapshot sync primary generation changed; resume on the selected storage') + } + return runPullSession( + { + reader, + writer, + mode: 'exclusive', + atomicCheckpoint: false, + activeStorage: direction === 'pull' ? this.getActiveUser().activeStorage : undefined, + loadRequest: () => + this.loadSyncRequest(this._authId, writer, readerSettings, writerSettings.storageIdentityKey), + commit: operation => operation() + }, + options + ) + }) + } + + private async trySnapshotCopy( + reader: sdk.WalletStorageSyncReader, + writer: sdk.WalletStorageProvider, + direction: 'push' | 'pull', + options: SyncSessionOptions = {}, + selection?: { generation: number; active: sdk.WalletStorageProvider } + ): Promise { + const selected = selection ?? { generation: this.generation, active: this.getActive() } + const source = reader.getSnapshotSync?.() + const destination = writer.getSnapshotSync?.() + if (source === undefined || destination === undefined || reader === writer) return undefined + if (!(await destination.supportsDestination())) return undefined + const cancelled = (): boolean => options.signal?.aborted === true + if (cancelled()) return { status: 'cancelled', mode: 'paged', pages: 0, inserts: 0, updates: 0 } + let view: WalletReadSnapshot | undefined + let partial: Pick = { + pages: 0, + inserts: 0, + updates: 0 + } + try { + view = await source.openSource(this._authId.identityKey, { + lifetimeMs: options.snapshotLifetimeMs, + signal: options.signal + }) + if (view === undefined) return undefined + let failed = false + try { + return await this.runSnapshotCopy( + view, + destination, + direction, + { + ...options, + onProgress: progress => { + partial = { + pages: progress.pages, + inserts: progress.inserts, + updates: progress.updates, + snapshotCheckpoint: + progress.snapshotCheckpoint === undefined + ? undefined + : { + ...progress.snapshotCheckpoint, + cursor: copySnapshotCursor(progress.snapshotCheckpoint.cursor) + } + } + options.onProgress?.(progress) + } + }, + selected.generation, + selected.active + ) + } catch (error) { + failed = true + throw error + } finally { + await view.close().catch(error => { + // A simultaneous expiry must not replace a malformed-row/session error. + // A physical cleanup failure, however, cannot be hidden by fallback. + if (source.fallbackOnResourceError === false || !failed || !(error instanceof SnapshotResourceLimitError)) + throw error + }) + } + } catch (error) { + if (error instanceof SnapshotCancelledError && cancelled()) + return { status: 'cancelled', mode: 'paged', ...partial } + // Existing ordinary backups must continue to accept large records and long + // copies. Only explicit resource limits select the established serialized + // fallback; corrupt rows, changed sessions and I/O failures still reject. + if (source.fallbackOnResourceError === false || !(error instanceof SnapshotResourceLimitError)) throw error + } + const result = await this.runLegacySnapshotFallback( + reader, + writer, + direction, + { + ...options, + onProgress: progress => { + options.onProgress?.({ + ...progress, + pages: partial.pages + progress.pages, + inserts: partial.inserts + progress.inserts, + updates: partial.updates + progress.updates + }) + } + }, + selected + ) + return { + ...result, + pages: partial.pages + result.pages, + inserts: partial.inserts + result.inserts, + updates: partial.updates + result.updates + } + } + async syncFromReader( identityKey: string, reader: sdk.WalletStorageSyncReader, activeSync?: sdk.WalletStorageSync, log: string = '' ): Promise<{ inserts: number; updates: number; log: string }> { + if (identityKey !== this._authId.identityKey) throw new WERR_UNAUTHORIZED() + const readerSettings = await reader.makeAvailable() + await this.preflightManagedNetworks(readerSettings) + if (activeSync != null) assertSyncNetwork(readerSettings, activeSync.getSettings()) const auth = await this.getAuth() - if (identityKey !== auth.identityKey) throw new WERR_UNAUTHORIZED() - const readerSettings = await reader.makeAvailable() + if (activeSync == null) { + const snapshot = await this.trySnapshotCopy(reader, this.getActive(), 'pull', { maxRoughSize: 10000000 }) + if (snapshot !== undefined) + return { + inserts: snapshot.inserts, + updates: snapshot.updates, + log: + log + `syncFromReader ${snapshot.mode} complete: ${snapshot.inserts} inserts, ${snapshot.updates} updates\n` + } + } let inserts = 0 let updates = 0 log = await this.runAsSync(async sync => { const writer = sync - const writerSettings = this.getSettings() + const writerSettings = writer.getSettings() + assertSyncNetwork(readerSettings, writerSettings) log += `syncFromReader from ${readerSettings.storageName} to ${writerSettings.storageName}\n` - const loadRequest = async (): Promise => - await this.loadSyncRequest(auth, writer, readerSettings, writerSettings.storageIdentityKey) + const loadRequest = (): Promise => + this.loadSyncRequest(auth, writer, readerSettings, writerSettings.storageIdentityKey) let args = await loadRequest() const budget = new SyncPageBudget() let i = -1 @@ -910,20 +1004,24 @@ export class WalletStorageManager implements sdk.WalletStorage { pageArgs.includeNextCheckpoint = true const startedAt = Date.now() const chunk = await reader.getSyncChunk(pageArgs) + const readMs = Date.now() - startedAt if (chunk.user != null) { // Merging state from a reader cannot update activeStorage chunk.user.activeStorage = ((this._active as ManagedStorage).user as TableUser).activeStorage } const r = await writer.processSyncChunk(pageArgs, chunk) - budget.committed(chunk, Date.now() - startedAt) + throwSyncResultError(r) + budget.committed(chunk, Date.now() - startedAt, readMs) inserts += r.inserts updates += r.updates log += `chunk ${i} inserted ${r.inserts} updated ${r.updates} ${String(r.maxUpdated_at)}\n` if (r.done) break - args = + const next = r.nextCheckpoint == null ? await loadRequest() : { ...args, ...validateSyncCheckpoint(r.nextCheckpoint, args) } + assertSyncProgress(args, next) + args = next } log += `syncFromReader complete: ${inserts} inserts, ${updates} updates\n` return log @@ -932,6 +1030,106 @@ export class WalletStorageManager implements sdk.WalletStorage { return { inserts, updates, log } } + /** + * Resumable pull with cancellation and per-page progress. Local providers + * advertising atomic checkpoints yield ownership during source I/O. Older + * and remote destinations keep the safe exclusive path. This is an eventual + * replica merge with a coherent source view when local SQL capabilities permit; + * it does not activate a primary or provide an archive snapshot of the destination. + */ + async syncFromReaderResumable( + identityKey: string, + reader: sdk.WalletStorageSyncReader, + options: SyncSessionOptions = {} + ): Promise { + if (identityKey !== this._authId.identityKey) throw new WERR_UNAUTHORIZED() + const readerSettings = await reader.makeAvailable() + await this.preflightManagedNetworks(readerSettings) + const auth = { ...(await this.getAuth()) } + const writer = this.getActive() + const writerSettings = writer.getSettings() + assertSyncNetwork(readerSettings, writerSettings) + const snapshot = await this.trySnapshotCopy(reader, writer, 'pull', options) + if (snapshot !== undefined) return snapshot + const generation = this.generation + const activeStorage = this.getActiveUser().activeStorage + const atomicCheckpoint = this._active?.access?.atomicSyncPages === true + const paged = + writer.isStorageProvider() && + atomicCheckpoint && + reader !== writer && + typeof (writer as Partial).prepareSyncChunk === 'function' + const assertCurrent = (): void => { + if (generation !== this.generation || writer !== this.getActive()) { + throw new WERR_INVALID_OPERATION( + 'Sync destination generation changed; resume on the selected storage provider.' + ) + } + } + const run = (commit: (operation: () => Promise) => Promise): Promise => + runPullSession( + { + reader, + writer, + activeStorage, + atomicCheckpoint, + mode: paged ? 'paged' : 'exclusive', + loadRequest: () => this.loadSyncRequest(auth, writer, readerSettings, writerSettings.storageIdentityKey), + prepare: paged ? (args, chunk) => (writer as StorageProvider).prepareSyncChunk(args, chunk) : undefined, + commit + }, + options + ) + if (paged) { + return await run(operation => + this.withAccess( + () => { + assertCurrent() + return operation() + }, + false, + true + ) + ) + } + return await this.runAsSync(() => { + assertCurrent() + return run(operation => { + assertCurrent() + return operation() + }) + }) + } + + /** Bounded push on negotiated local snapshots, with the legacy serialized fallback. */ + async syncToWriterResumable( + auth: sdk.AuthId, + writer: sdk.WalletStorageProvider, + options: SyncSessionOptions = {} + ): Promise { + if (auth.identityKey !== this._authId.identityKey) throw new WERR_UNAUTHORIZED() + const writerSettings = await writer.makeAvailable() + await this.preflightManagedNetworks(writerSettings) + await this.getAuth() + const snapshot = await this.trySnapshotCopy(this.getActive(), writer, 'push', options) + if (snapshot !== undefined) return snapshot + return await this.runAsSync(async reader => { + const settings = reader.getSettings() + assertSyncNetwork(settings, writerSettings) + return await runPullSession( + { + reader, + writer, + atomicCheckpoint: false, + mode: 'exclusive', + loadRequest: () => this.loadSyncRequest(auth, writer, settings, writerSettings.storageIdentityKey), + commit: operation => operation() + }, + options + ) + }) + } + async syncToWriter( auth: sdk.AuthId, writer: sdk.WalletStorageProvider, @@ -939,9 +1137,32 @@ export class WalletStorageManager implements sdk.WalletStorage { log: string = '', progLog?: (s: string) => string ): Promise<{ inserts: number; updates: number; log: string }> { + if (auth.identityKey !== this._authId.identityKey) throw new WERR_UNAUTHORIZED() progLog ||= s => s const writerSettings = await writer.makeAvailable() + await this.preflightManagedNetworks(writerSettings) + if (activeSync != null) assertSyncNetwork(activeSync.getSettings(), writerSettings) + + if (activeSync == null) { + await this.getAuth() + const snapshot = await this.trySnapshotCopy(this.getActive(), writer, 'push', { + maxRoughSize: 10000000, + onProgress: progress => { + log += this.committedPageLog(progress, progLog) + } + }) + if (snapshot !== undefined) + return { + inserts: snapshot.inserts, + updates: snapshot.updates, + log: + log + + progLog( + `syncToWriter ${snapshot.mode} complete: ${snapshot.inserts} inserts, ${snapshot.updates} updates\n` + ) + } + } let inserts = 0 let updates = 0 @@ -949,11 +1170,12 @@ export class WalletStorageManager implements sdk.WalletStorage { log = await this.runAsSync(async sync => { const reader = sync const readerSettings = reader.getSettings() + assertSyncNetwork(readerSettings, writerSettings) log += progLog(`syncToWriter from ${readerSettings.storageName} to ${writerSettings.storageName}\n`) - const loadRequest = async (): Promise => - await this.loadSyncRequest(auth, writer, readerSettings, writerSettings.storageIdentityKey) + const loadRequest = (): Promise => + this.loadSyncRequest(auth, writer, readerSettings, writerSettings.storageIdentityKey) let args = await loadRequest() const budget = new SyncPageBudget() let i = -1 @@ -965,17 +1187,21 @@ export class WalletStorageManager implements sdk.WalletStorage { pageArgs.includeNextCheckpoint = true const startedAt = Date.now() const chunk = await reader.getSyncChunk(pageArgs) + const readMs = Date.now() - startedAt log += EntitySyncState.syncChunkSummary(chunk) const r = await writer.processSyncChunk(pageArgs, chunk) - budget.committed(chunk, Date.now() - startedAt) + throwSyncResultError(r) + budget.committed(chunk, Date.now() - startedAt, readMs) inserts += r.inserts updates += r.updates log += progLog(`chunk ${i} inserted ${r.inserts} updated ${r.updates} ${String(r.maxUpdated_at)}\n`) if (r.done) break - args = + const next = r.nextCheckpoint == null ? await loadRequest() : { ...args, ...validateSyncCheckpoint(r.nextCheckpoint, args) } + assertSyncProgress(args, next) + args = next } log += progLog(`syncToWriter complete: ${inserts} inserts, ${updates} updates\n`) return log @@ -987,6 +1213,27 @@ export class WalletStorageManager implements sdk.WalletStorage { async updateBackups(activeSync?: sdk.WalletStorageSync, progLog?: (s: string) => string): Promise { progLog ||= s => s const auth = await this.getAuth(true) + if (activeSync == null) { + const selected = { generation: this.generation, active: this.getActive() } + const backups = [...(this._backups as ManagedStorage[])] + let log = progLog(`BACKUP CURRENT ACTIVE TO ${backups.length} STORES\n`) + // One source view at a time; stop before starting another backup on failure. + await runInSeries(backups, async backup => { + const options: SyncSessionOptions = { + maxRoughSize: 10000000, + onProgress: progress => { + log += this.committedPageLog(progress, progLog) + } + } + const result = + (await this.trySnapshotCopy(selected.active, backup.storage, 'push', options, selected)) ?? + (await this.runLegacySnapshotFallback(selected.active, backup.storage, 'push', options, selected)) + log += progLog( + `${result.mode === 'paged' ? 'snapshot' : 'serialized'} complete: ${result.inserts} inserts, ${result.updates} updates\n` + ) + }) + return log + } return await this.runAsSync(async sync => { let log = progLog(`BACKUP CURRENT ACTIVE TO ${(this._backups as ManagedStorage[]).length} STORES\n`) for (const backup of this._backups as ManagedStorage[]) { @@ -1024,84 +1271,92 @@ export class WalletStorageManager implements sdk.WalletStorage { let log = progLog(`setActive to ${(newActive.settings as TableSettings).storageName}`) - if (storageIdentityKey === this.getActiveStore() && this.isActiveEnabled) { - /** Setting the current active as the new active is a permitted no-op. */ - return log + progLog(' unchanged\n') - } - - log += progLog('\n') - log += await this.runAsSync(async _sync => { - let log = '' - - if ((this._conflictingActives as ManagedStorage[]).length > 0) { - // Merge state from conflicting actives into `newActive`. - - // Handle case where new active is current active to resolve conflicts. - // And where new active is one of the current conflict actives. - ;(this._conflictingActives as ManagedStorage[]).push(this._active as ManagedStorage) - // Remove the new active from conflicting actives and - // set new active as the conflicting active that matches the target `storageIdentityKey` - this._conflictingActives = (this._conflictingActives as ManagedStorage[]).filter(ca => { - const isNewActive = (ca.settings as TableSettings).storageIdentityKey === storageIdentityKey - return !isNewActive - }) - - // Merge state from conflicting actives into `newActive`. - for (const conflict of this._conflictingActives) { - log += progLog('MERGING STATE FROM CONFLICTING ACTIVES:\n') - const sfr = await this.syncToWriter( - { identityKey, userId: (newActive.user as TableUser).userId, isActive: false }, - newActive.storage, - conflict.storage, - undefined, - progLog - ) - log += sfr.log - } - log += progLog('PROPAGATE MERGED ACTIVE STATE TO NON-ACTIVES\n') - } else { - log += progLog('BACKUP CURRENT ACTIVE STATE THEN SET NEW ACTIVE\n') + // An earlier queued switch may have changed the primary since this call. + // Decide even a no-op under the same ownership as an actual switch. + if (storageIdentityKey === this.getActiveStore() && this.isActiveEnabled) { + return progLog(' unchanged\n') } - - // If there were conflicting actives, - // Push state merged from all merged actives into newActive to all stores other than the now single active. - // Otherwise, - // Push state from current active to all other stores. - const backupSource = - (this._conflictingActives as ManagedStorage[]).length > 0 ? newActive : (this._active as ManagedStorage) - - // Update the backupSource's user record with the new activeStorage - // which will propagate to all other stores in the following backup loop. - await backupSource.storage.setActive( - { identityKey, userId: (backupSource.user as TableUser).userId }, - storageIdentityKey - ) - - for (const store of this._stores) { - // Update cached user.activeStorage of all stores - ;(store.user as TableUser).activeStorage = storageIdentityKey - - if ( - (store.settings as TableSettings).storageIdentityKey !== - (backupSource.settings as TableSettings).storageIdentityKey - ) { - // If this store is not the backupSource store push state from backupSource to this store. - const stwr = await this.syncToWriter( - { identityKey, userId: (store.user as TableUser).userId, isActive: false }, - store.storage, - backupSource.storage, - undefined, - progLog - ) - log += stwr.log + let log = progLog('\n') + this.generation++ + + try { + if ((this._conflictingActives as ManagedStorage[]).length > 0) { + // Merge state from conflicting actives into `newActive`. + + // Handle case where new active is current active to resolve conflicts. + // And where new active is one of the current conflict actives. + ;(this._conflictingActives as ManagedStorage[]).push(this._active as ManagedStorage) + // Remove the new active from conflicting actives and + // set new active as the conflicting active that matches the target `storageIdentityKey` + this._conflictingActives = (this._conflictingActives as ManagedStorage[]).filter(ca => { + const isNewActive = (ca.settings as TableSettings).storageIdentityKey === storageIdentityKey + return !isNewActive + }) + + // Merge state from conflicting actives into `newActive`. + for (const conflict of this._conflictingActives) { + log += progLog('MERGING STATE FROM CONFLICTING ACTIVES:\n') + const sfr = await this.syncToWriter( + { identityKey, userId: (newActive.user as TableUser).userId, isActive: false }, + newActive.storage, + conflict.storage, + undefined, + progLog + ) + log += sfr.log + } + log += progLog('PROPAGATE MERGED ACTIVE STATE TO NON-ACTIVES\n') + } else { + log += progLog('BACKUP CURRENT ACTIVE STATE THEN SET NEW ACTIVE\n') } - } - this._isAvailable = false - await this.makeAvailable() + // If there were conflicting actives, + // Push state merged from all merged actives into newActive to all stores other than the now single active. + // Otherwise, + // Push state from current active to all other stores. + const backupSource = + (this._conflictingActives as ManagedStorage[]).length > 0 ? newActive : (this._active as ManagedStorage) + + // Update the backupSource's user record with the new activeStorage + // which will propagate to all other stores in the following backup loop. + await backupSource.storage.setActive( + { identityKey, userId: (backupSource.user as TableUser).userId }, + storageIdentityKey + ) + + for (const store of this._stores) { + // Update cached user.activeStorage of all stores + ;(store.user as TableUser).activeStorage = storageIdentityKey + + if ( + (store.settings as TableSettings).storageIdentityKey !== + (backupSource.settings as TableSettings).storageIdentityKey + ) { + // If this store is not the backupSource store push state from backupSource to this store. + const stwr = await this.syncToWriter( + { identityKey, userId: (store.user as TableUser).userId, isActive: false }, + store.storage, + backupSource.storage, + undefined, + progLog + ) + log += stwr.log + } + } - return log + this._isAvailable = false + await this.makeAvailable() + + return log + } catch (error) { + // Partial propagation can leave stores disagreeing about the primary. + // Discard cached users before the next authorization or queued access. + this._isAvailable = false + this._authId.isActive = false + for (const store of this._stores) store.isAvailable = false + throw error + } }) return log diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/StorageKnexMigrationFailure.security.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/StorageKnexMigrationFailure.security.test.ts index 913e69db0..72caa4ff5 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/StorageKnexMigrationFailure.security.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/StorageKnexMigrationFailure.security.test.ts @@ -1,4 +1,11 @@ import { StorageKnex } from '../StorageKnex' +import { KnexMigrations } from '../schema/KnexMigrations' + +function absentGeneration() { + return jest.fn(async (scope: (trx: object) => Promise) => { + await scope({ schema: { hasTable: jest.fn().mockResolvedValue(false) } }) + }) +} function storageWithKnex(knex: object): StorageKnex { return { knex, chain: 'test' } as unknown as StorageKnex @@ -24,6 +31,7 @@ describe('StorageKnex migration failure boundaries', () => { test('dropAllData stops only at the explicit empty-schema state', async () => { const knex = { client: { config: { client: 'better-sqlite3' } }, + transaction: absentGeneration(), raw: jest.fn().mockResolvedValue(undefined), migrate: { currentVersion: jest.fn().mockResolvedValueOnce('202609170001').mockResolvedValueOnce('none'), @@ -46,6 +54,7 @@ describe('StorageKnex migration failure boundaries', () => { ])('propagates when rollback %s and always restores SQLite foreign keys', async (_name, down) => { const knex = { client: { config: { client: 'better-sqlite3' } }, + transaction: absentGeneration(), raw: jest.fn().mockResolvedValue(undefined), migrate: { currentVersion: jest.fn().mockResolvedValue('202609170001'), @@ -53,13 +62,16 @@ describe('StorageKnex migration failure boundaries', () => { } } - await expect(StorageKnex.prototype.dropAllData.call(storageWithKnex(knex))).rejects.toThrow() + await expect(StorageKnex.prototype.dropAllData.call(storageWithKnex(knex))).rejects.toThrow( + _name === 'throws' ? 'rollback failed' : 'database migration rollback returned no result' + ) expect(knex.raw).toHaveBeenLastCalledWith('PRAGMA foreign_keys = ON;') }) test('migrate restores SQLite foreign keys when migration fails', async () => { const knex = { client: { config: { client: 'better-sqlite3' } }, + transaction: absentGeneration(), raw: jest.fn().mockResolvedValue(undefined), migrate: { latest: jest.fn().mockRejectedValue(new Error('migration failed')), @@ -73,3 +85,35 @@ describe('StorageKnex migration failure boundaries', () => { expect(knex.raw).toHaveBeenLastCalledWith('PRAGMA foreign_keys = ON;') }) }) + +test('dropAllData bounds a migrator that never reaches the empty schema', async () => { + const database = { + client: { config: { client: 'better-sqlite3' } }, + transaction: absentGeneration(), + raw: jest.fn().mockResolvedValue(undefined), + migrate: { + currentVersion: jest.fn().mockResolvedValue('unfinished'), + down: jest.fn().mockResolvedValue([1, ['fixture']]) + } + } + await expect(StorageKnex.prototype.dropAllData.call(storageWithKnex(database))).rejects.toThrow( + 'database migration rollback did not reach the empty schema' + ) + expect(database.migrate.down).toHaveBeenCalledTimes( + Object.keys(new KnexMigrations('test', '', '', 1024).migrations).length + ) + expect(database.raw).toHaveBeenLastCalledWith('PRAGMA foreign_keys = ON;') +}) + +test('MySQL full deletion does not issue SQLite generation operations', async () => { + const database = { + client: { config: { client: 'mysql2' } }, + transaction: jest.fn(), + raw: jest.fn(), + migrate: { currentVersion: jest.fn().mockResolvedValue('none'), down: jest.fn() } + } + await StorageKnex.prototype.dropAllData.call(storageWithKnex(database)) + expect(database.transaction).not.toHaveBeenCalled() + expect(database.raw).not.toHaveBeenCalled() + expect(database.migrate.down).not.toHaveBeenCalled() +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/__test/WalletStorageManager.test.ts b/packages/wallet/wallet-toolbox/src/storage/__test/WalletStorageManager.test.ts index 3ed225f4d..5d6e01095 100644 --- a/packages/wallet/wallet-toolbox/src/storage/__test/WalletStorageManager.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/__test/WalletStorageManager.test.ts @@ -3,9 +3,35 @@ import { wait } from '../..' import { _tu, TestWalletNoSetup } from '../../../test/utils/TestUtilsWalletStorage' import { StorageProvider } from '../StorageProvider' import { StorageReaderWriter } from '../StorageReaderWriter' +import { TableProvenTx } from '../schema/tables' +import { toBinaryBaseBlockHeader } from '../../services/Services' +import { doubleSha256BE } from '../../utility/utilityHelpers' +import { asString } from '../../utility/utilityHelpers.noBuffer' import * as dotenv from 'dotenv' +function canonicalReproof(ctx: TestWalletNoSetup, ptx: TableProvenTx) { + const merklePath = new bsv.MerklePath(ptx.height + 1, [[{ offset: 0, hash: ptx.txid, txid: true }]]) + const header = toBinaryBaseBlockHeader({ + version: 1, + previousHash: '0'.repeat(64), + merkleRoot: merklePath.computeRoot(ptx.txid), + time: 0, + bits: 0, + nonce: 0 + }) + const services = ctx.storage.getServices() + const isValidRootForHeight = jest.fn(async () => true) + jest.spyOn(services, 'getChainTracker').mockResolvedValue({ isValidRootForHeight } as bsv.ChainTracker) + jest.spyOn(services, 'getHeaderForHeight').mockResolvedValue(header) + const lookup = jest.spyOn(services, 'getValidatedMerklePath').mockImplementation(async (_txid, validate) => { + const result = { name: 'canonical reproof fixture', merklePath } + await validate(result) + return result + }) + return { lookup, isValidRootForHeight, height: merklePath.blockHeight, blockHash: asString(doubleSha256BE(header)) } +} + dotenv.config() describe('WalletStorageManager tests', () => { jest.setTimeout(99999999) @@ -63,6 +89,195 @@ describe('WalletStorageManager tests', () => { } }) + test.each(['runAsReader', 'runAsWriter', 'runAsStorageProvider'] as const)( + '%s returns a rejection for synchronous callback errors and releases ownership', + async method => { + for (const { storage } of ctxs) { + const failure = new Error('synchronous callback failure') + const operation = storage[method](() => { + throw failure + }) + expect(operation).toBeInstanceOf(Promise) + await expect(operation).rejects.toBe(failure) + await expect(storage.runAsWriter(async () => 'next writer')).resolves.toBe('next writer') + } + } + ) + + test.each(['runAsReader', 'runAsWriter', 'runAsStorageProvider'] as const)( + '%s holds ownership through asynchronous rejection, then releases the waiting writer', + async method => { + for (const { storage } of ctxs) { + let reject!: (error: Error) => void + let entered!: () => void + const started = new Promise(resolve => { + entered = resolve + }) + const pending = new Promise((_resolve, fail) => { + reject = fail + }) + const operation = storage[method](() => { + entered() + return pending + }) + await started + const next = jest.fn(async () => 'next writer') + const waiting = storage.runAsWriter(next) + await Promise.resolve() + expect(next).not.toHaveBeenCalled() + const failure = new Error('asynchronous callback failure') + const rejected = expect(operation).rejects.toBe(failure) + reject(failure) + await rejected + await expect(waiting).resolves.toBe('next writer') + expect(next).toHaveBeenCalledTimes(1) + } + } + ) + + test.each(['migrate', 'getCapabilities', 'findProvenTxReqs'] as const)( + '%s preserves synchronous provider failures as rejected promises and releases ownership', + async method => { + const { storage } = ctxs[0] + const failure = new Error('provider failed synchronously') + const dispatch = jest.spyOn(storage.getActive(), method).mockImplementation(() => { + throw failure + }) + try { + const args = method === 'migrate' ? ['name', 'identity'] : method === 'getCapabilities' ? [] : [{}] + const operation = Reflect.get(storage, method).apply(storage, args) + expect(operation).toBeInstanceOf(Promise) + await expect(operation).rejects.toBe(failure) + expect(dispatch).toHaveBeenCalledTimes(1) + await expect(storage.runAsWriter(async () => 'next writer')).resolves.toBe('next writer') + } finally { + dispatch.mockRestore() + } + } + ) + + test('writer authorization failure never dispatches and releases the next writer', async () => { + for (const { storage } of ctxs) { + const failure = new Error('authorization unavailable') + const auth = jest.spyOn(storage, 'getAuth').mockRejectedValueOnce(failure) + const dispatch = jest.spyOn(storage.getActive(), 'createAction') + try { + await expect(storage.createAction({} as any)).rejects.toBe(failure) + expect(auth).toHaveBeenCalledWith(true) + expect(dispatch).not.toHaveBeenCalled() + await expect(storage.runAsWriter(async () => 'next writer')).resolves.toBe('next writer') + } finally { + auth.mockRestore() + dispatch.mockRestore() + } + } + }) + + test('writer authorization is evaluated only after the preceding writer releases ownership', async () => { + const { storage } = ctxs[0] + let release!: () => void + let entered!: () => void + const started = new Promise(resolve => { + entered = resolve + }) + const held = storage.runAsWriter(async () => { + entered() + await new Promise(resolve => { + release = resolve + }) + }) + await started + const auth = jest.spyOn(storage, 'getAuth') + const failure = new Error('provider refused action') + const dispatch = jest.spyOn(storage.getActive(), 'createAction').mockImplementation(() => { + throw failure + }) + try { + const action = storage.createAction({} as any) + const rejected = expect(action).rejects.toBe(failure) + await Promise.resolve() + expect(auth).not.toHaveBeenCalled() + expect(dispatch).not.toHaveBeenCalled() + release() + await held + await rejected + expect(auth).toHaveBeenCalledWith(true) + expect(dispatch).toHaveBeenCalledTimes(1) + await expect(storage.runAsWriter(async () => 'next writer')).resolves.toBe('next writer') + } finally { + release() + await held + auth.mockRestore() + dispatch.mockRestore() + } + }) + + test('reader authorization still precedes queue admission while provider work waits for ownership', async () => { + const { storage } = ctxs[0] + let release!: () => void + let entered!: () => void + const started = new Promise(resolve => { + entered = resolve + }) + const held = storage.runAsWriter(async () => { + entered() + await new Promise(resolve => { + release = resolve + }) + }) + await started + const auth = jest.spyOn(storage, 'getAuth') + const failure = new Error('reader provider refused request') + const dispatch = jest.spyOn(storage.getActive(), 'findOutputsAuth').mockImplementation(() => { + throw failure + }) + try { + const result = storage.findOutputs({} as any) + const rejected = expect(result).rejects.toBe(failure) + expect(auth).toHaveBeenCalledWith() + await Promise.resolve() + expect(dispatch).not.toHaveBeenCalled() + release() + await held + await rejected + expect(dispatch).toHaveBeenCalledTimes(1) + await expect(storage.runAsWriter(async () => 'next writer')).resolves.toBe('next writer') + } finally { + release() + await held + auth.mockRestore() + dispatch.mockRestore() + } + }) + + test('reader authorization rejection never dispatches provider work', async () => { + const { storage } = ctxs[0] + const failure = new Error('reader authorization unavailable') + const auth = jest.spyOn(storage, 'getAuth').mockRejectedValueOnce(failure) + const dispatch = jest.spyOn(storage.getActive(), 'findOutputsAuth') + try { + const result = storage.findOutputs({} as any) + expect(result).toBeInstanceOf(Promise) + await expect(result).rejects.toBe(failure) + expect(dispatch).not.toHaveBeenCalled() + await expect(storage.runAsWriter(async () => 'next writer')).resolves.toBe('next writer') + } finally { + auth.mockRestore() + dispatch.mockRestore() + } + }) + + test('borrowed sync callback throws remain asynchronous rejections', async () => { + const { storage } = ctxs[0] + const failure = new Error('borrowed callback failed') + const operation = storage.runAsSync(() => { + throw failure + }, storage.getActive()) + expect(operation).toBeInstanceOf(Promise) + await expect(operation).rejects.toBe(failure) + await expect(storage.runAsWriter(async () => 'next writer')).resolves.toBe('next writer') + }) + test('1_runAsReader runAsWriter runAsSync interlock correctly', async () => { const { storage } = await _tu.createSQLiteTestSetup1Wallet({ databaseName: 'syncTest1' @@ -478,16 +693,11 @@ describe('WalletStorageManager tests', () => { const [ptx] = await ctx.activeStorage.findProvenTxs({ partial: {} }) expect(ptx).toBeTruthy() const epoch = await ctx.activeStorage.readPreparedBeefProofEpoch() - const reprove = jest.spyOn(ctx.storage, 'reproveProven').mockResolvedValue({ - log: '', - updated: { update: { height: ptx.height }, logUpdate: '' }, - unchanged: false, - unavailable: false - }) + const { lookup } = canonicalReproof(ctx, ptx) const result = await ctx.storage.reproveHeader(ptx.blockHash) - expect(reprove).toHaveBeenCalled() + expect(lookup).toHaveBeenCalled() expect(result.updated.length).toBeGreaterThan(0) await expect(ctx.activeStorage.readPreparedBeefProofEpoch()).resolves.toBe(epoch + 1) } finally { @@ -539,12 +749,8 @@ describe('WalletStorageManager tests', () => { const [ptx] = await ctx.activeStorage.findProvenTxs({ partial: {} }) expect(ptx).toBeTruthy() const epoch = await ctx.activeStorage.readPreparedBeefProofEpoch() - jest.spyOn(ctx.storage, 'reproveProven').mockResolvedValue({ - log: '', - updated: undefined, - unchanged: false, - unavailable: true - }) + const { lookup } = canonicalReproof(ctx, ptx) + lookup.mockRejectedValue(new Error('proof unavailable')) const result = await ctx.storage.reproveHeader(ptx.blockHash) @@ -562,16 +768,11 @@ describe('WalletStorageManager tests', () => { const [ptx] = await ctx.activeStorage.findProvenTxs({ partial: {} }) expect(ptx).toBeTruthy() const epoch = await ctx.activeStorage.readPreparedBeefProofEpoch() - const reprove = jest.spyOn(ctx.storage, 'reproveProven').mockResolvedValue({ - log: '', - updated: { update: { height: ptx.height }, logUpdate: 'height reproof\n' }, - unchanged: false, - unavailable: false - }) + const { lookup } = canonicalReproof(ctx, ptx) const result = await ctx.storage.reproveHeightMerkleRoot(ptx.height, ptx.merkleRoot) - expect(reprove).toHaveBeenCalledWith(ptx, true) + expect(lookup).toHaveBeenCalledWith(ptx.txid, expect.any(Function)) expect(result.updated).toHaveLength(1) expect(result.log).toContain('proof data updated') await expect(ctx.activeStorage.readPreparedBeefProofEpoch()).resolves.toBe(epoch + 1) @@ -592,36 +793,8 @@ describe('WalletStorageManager tests', () => { const [ptx] = await ctx.activeStorage.findProvenTxs({ partial: {} }) expect(ptx).toBeTruthy() const epoch = await ctx.activeStorage.readPreparedBeefProofEpoch() - const replacementHash = ptx.blockHash === 'f'.repeat(64) ? 'e'.repeat(64) : 'f'.repeat(64) - const replacementHeight = ptx.height + 1 - const merklePath = new bsv.MerklePath(replacementHeight, [ - [ - { - offset: 0, - hash: ptx.txid, - txid: true - } - ] - ]) - const services = ctx.storage.getServices() - const isValidRootForHeight = jest.fn(async () => 'true' as unknown as boolean) - jest.spyOn(services, 'getChainTracker').mockResolvedValue({ - isValidRootForHeight - } as bsv.ChainTracker) - jest.spyOn(services, 'getMerklePath').mockResolvedValue({ - name: 'prepared BEEF reproof test', - merklePath, - header: { - version: 1, - previousHash: '0'.repeat(64), - merkleRoot: merklePath.computeRoot(ptx.txid), - time: 0, - bits: 0, - nonce: 0, - height: replacementHeight, - hash: replacementHash - } - }) + const { height: replacementHeight, blockHash: replacementHash, isValidRootForHeight } = canonicalReproof(ctx, ptx) + isValidRootForHeight.mockResolvedValue('true' as unknown as boolean) const rejected = await ctx.storage.reproveProven(ptx) expect(rejected).toMatchObject({ unavailable: true, updated: undefined }) diff --git a/packages/wallet/wallet-toolbox/src/storage/index.all.ts b/packages/wallet/wallet-toolbox/src/storage/index.all.ts index be16c7f3b..6af578d85 100644 --- a/packages/wallet/wallet-toolbox/src/storage/index.all.ts +++ b/packages/wallet/wallet-toolbox/src/storage/index.all.ts @@ -1,6 +1,7 @@ export * from './WalletStorageManager' export * from './StorageProvider' export * from './StorageSyncReader' +export type { SyncSessionOptions, SyncSessionProgress, SyncSessionResult } from './sync/syncSession' export * from './remoting/StorageClient' export * from './remoting/StorageServer' export * from './remoting/KnexSessionManager' @@ -19,3 +20,26 @@ export * from './schema/tables/index' export * from './schema/entities/index' export * as sync from './sync' export * from './portable' + +export type { RetainedReadSnapshot, RetainedReadSnapshotOptions } from './snapshot/RetainedReadSnapshot' + +export type { + PackedSnapshotRow, + WalletReadSnapshot, + WalletReadSnapshotOptions, + WalletSnapshotArchivePosition, + WalletSnapshotCursor, + WalletSnapshotPage, + WalletSnapshotPageLimits, + WalletSnapshotTable, + WalletSnapshotTables +} from './snapshot/WalletReadSnapshot' + +export { snapshotSyncTables } from './snapshot/SnapshotSync' +export type { + SnapshotSyncSource, + SnapshotSyncCheckpoint, + SnapshotSyncCommit, + SnapshotSyncStorage, + SnapshotSyncTable +} from './snapshot/SnapshotSync' diff --git a/packages/wallet/wallet-toolbox/src/storage/index.client.ts b/packages/wallet/wallet-toolbox/src/storage/index.client.ts index 89419312b..6b25444c0 100644 --- a/packages/wallet/wallet-toolbox/src/storage/index.client.ts +++ b/packages/wallet/wallet-toolbox/src/storage/index.client.ts @@ -2,6 +2,7 @@ export * from './WalletStorageManager' export * from './StorageIdb' export * from './StorageProvider' export * from './StorageSyncReader' +export type { SyncSessionOptions, SyncSessionProgress, SyncSessionResult } from './sync/syncSession' export * from './schema/tables/index' export * from './schema/entities/index' export * from './remoting/StorageClient' @@ -10,3 +11,26 @@ export * from './methods/ListActionsSpecOp' export * from './methods/ListOutputsSpecOp' export * from './methods/managedChange' export * from './methods/managedChangePolicy' + +export type { RetainedReadSnapshot, RetainedReadSnapshotOptions } from './snapshot/RetainedReadSnapshot' + +export type { + PackedSnapshotRow, + WalletReadSnapshot, + WalletReadSnapshotOptions, + WalletSnapshotArchivePosition, + WalletSnapshotCursor, + WalletSnapshotPage, + WalletSnapshotPageLimits, + WalletSnapshotTable, + WalletSnapshotTables +} from './snapshot/WalletReadSnapshot' + +export { snapshotSyncTables } from './snapshot/SnapshotSync' +export type { + SnapshotSyncSource, + SnapshotSyncCheckpoint, + SnapshotSyncCommit, + SnapshotSyncStorage, + SnapshotSyncTable +} from './snapshot/SnapshotSync' diff --git a/packages/wallet/wallet-toolbox/src/storage/index.mobile.ts b/packages/wallet/wallet-toolbox/src/storage/index.mobile.ts index 0a709e46a..60b1a3133 100644 --- a/packages/wallet/wallet-toolbox/src/storage/index.mobile.ts +++ b/packages/wallet/wallet-toolbox/src/storage/index.mobile.ts @@ -1,6 +1,7 @@ export * from './WalletStorageManager' export * from './StorageProvider' export * from './StorageSyncReader' +export type { SyncSessionOptions, SyncSessionProgress, SyncSessionResult } from './sync/syncSession' export * from './schema/tables/index' export * from './schema/entities/index' export * from './remoting/StorageMobile' @@ -8,3 +9,26 @@ export * from './portable' export * from './methods/ListActionsSpecOp' export * from './methods/ListOutputsSpecOp' export * from './methods/managedChangePolicy' + +export type { RetainedReadSnapshot, RetainedReadSnapshotOptions } from './snapshot/RetainedReadSnapshot' + +export type { + PackedSnapshotRow, + WalletReadSnapshot, + WalletReadSnapshotOptions, + WalletSnapshotArchivePosition, + WalletSnapshotCursor, + WalletSnapshotPage, + WalletSnapshotPageLimits, + WalletSnapshotTable, + WalletSnapshotTables +} from './snapshot/WalletReadSnapshot' + +export { snapshotSyncTables } from './snapshot/SnapshotSync' +export type { + SnapshotSyncSource, + SnapshotSyncCheckpoint, + SnapshotSyncCommit, + SnapshotSyncStorage, + SnapshotSyncTable +} from './snapshot/SnapshotSync' diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/attemptToPostReqsToNetwork.ts b/packages/wallet/wallet-toolbox/src/storage/methods/attemptToPostReqsToNetwork.ts index 1ea521444..bd968f73c 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/attemptToPostReqsToNetwork.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/attemptToPostReqsToNetwork.ts @@ -1,4 +1,5 @@ import { Beef, Transaction, WalletLoggerInterface } from '@bsv/sdk' +import { repairBeefProofs } from './repairBeefProofs' import { StorageProvider } from '../StorageProvider' import { EntityProvenTxReq } from '../schema/entities' import type * as sdk from '../../sdk' @@ -28,6 +29,9 @@ export async function attemptToPostReqsToNetwork( const services = storage.getServices() + // Rebuilding stored input BEEF may reintroduce an orphaned ancestor even + // after an earlier foreground check. This is also the monitor's send path. + r.beef = await repairBeefProofs(storage, r.beef, trx) const pbrs = await services.postBeef(r.beef, txids, logger) // post beef results (pbrs) is an array by service provider diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts index 2725c5a98..7b5e6e0a7 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/createAction.ts @@ -5,6 +5,7 @@ import { validateSatoshis } from '@bsv/sdk/wallet/validationHelpers' import { Beef, OriginatorDomainNameStringUnder250Bytes, Random, Script, TelemetrySpan } from '@bsv/sdk' +import { repairBeefProofs } from './repairBeefProofs' import { toBase64 } from '@bsv/sdk/primitives/utils' import { generateChangeSdk, @@ -2003,6 +2004,11 @@ async function mergeAllocatedChangeBeefs( }) } ) + // Standalone offline storage historically permits construction without + // services. Configured wallets validate even prepared/embedded ancestry + // before returning proofs to their signer; offline callers retain their + // existing responsibility to validate before signing or broadcasting. + if (storage._services != null) beef = await repairBeefProofs(storage, beef) const inputBeef = await traceStorageStep( storage, 'wallet.storage.create_action.beef_trim_serialize', diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts b/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts index 489646527..333b53db3 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/processAction.ts @@ -1,3 +1,4 @@ +import { repairBeefProofs } from './repairBeefProofs' // eslint-disable-next-line @typescript-eslint/no-unused-vars import { Beef, @@ -45,7 +46,7 @@ interface PreparedBeefWriterExtension { enqueuePreparedBeef: (preparation: PreparedBeefPreparation) => boolean } -function normalizeProcessActionArgs (args: StorageProcessActionArgs): StorageProcessActionArgs { +function normalizeProcessActionArgs(args: StorageProcessActionArgs): StorageProcessActionArgs { for (const property of ['isNewTx', 'isSendWith', 'isNoSend', 'isDelayed'] as const) { if (typeof args[property] !== 'boolean') { throw new WERR_INVALID_PARAMETER(property, 'a boolean') @@ -67,7 +68,7 @@ function normalizeProcessActionArgs (args: StorageProcessActionArgs): StoragePro return { ...args, sendWith } } -export async function processAction ( +export async function processAction( storage: StorageProvider, auth: AuthId, args: StorageProcessActionArgs @@ -94,7 +95,7 @@ export async function processAction ( ) } -async function processActionCore ( +async function processActionCore( storage: StorageProvider, auth: AuthId, args: StorageProcessActionArgs, @@ -133,7 +134,9 @@ async function processActionCore ( typeof preparedBeef.preparedBeefWritesEnabled === 'function' && preparedBeef.preparedBeefWritesEnabled.call(storage) && vargs.outputOutputs.some(isManagedChangeOutput) - ) { prepareRootTxid = req.txid } + ) { + prepareRootTxid = req.txid + } // Add the new txid to sendWith unless there are no others to send and the noSend option is set. if (args.isNoSend && !args.isSendWith) { logger?.log(`noSend txid ${req.txid}`) @@ -169,7 +172,7 @@ async function processActionCore ( return r } -async function traceProcessStep ( +async function traceProcessStep( storage: StorageProvider, name: string, parent: TelemetrySpan | undefined, @@ -225,7 +228,7 @@ export interface PostBeefResultForTxidApi { * @param isDelayed * @param r Optional. Ignores txids and allows ProvenTxReqs and merged beef to be passed in. */ -function classifyReqDetails ( +function classifyReqDetails( details: GetReqsAndBeefDetail[], swr: SendWithResult[], readyToSendReqs: EntityProvenTxReq[] @@ -247,22 +250,23 @@ function classifyReqDetails ( return complete } -async function verifyMergedBeef ( +async function verifyMergedBeef( storage: StorageProvider, r: GetReqsAndBeefResult, readyToSendReqs: EntityProvenTxReq[], logger?: WalletLoggerInterface ): Promise { if (readyToSendReqs.length === 0 || r.verified === true) return + r.beef = await repairBeefProofs(storage, r.beef) const beefIsValid = await r.beef.verify(await storage.getServices().getChainTracker()) if (!beefIsValid) { - logger?.error(`VERIFY FALSE BEEF: ${r.beef.toLogString()}`) + logger?.error('Merged transaction proof graph failed validation; no broadcast attempted.') throw new WERR_INTERNAL('merged Beef failed validation.') } logger?.log('beef is valid') } -async function getReqDetailsForDelayedShare (storage: StorageProvider, txids: string[]): Promise { +async function getReqDetailsForDelayedShare(storage: StorageProvider, txids: string[]): Promise { const r: GetReqsAndBeefResult = { beef: new Beef(), details: [] @@ -298,14 +302,14 @@ async function getReqDetailsForDelayedShare (storage: StorageProvider, txids: st return r } -export async function shareReqsWithWorld ( +export async function shareReqsWithWorld( storage: StorageProvider, userId: number, txids: string[], isDelayed: boolean, r?: GetReqsAndBeefResult, logger?: WalletLoggerInterface -): Promise<{ swr: SendWithResult[], ndr: ReviewActionResult[] | undefined }> { +): Promise<{ swr: SendWithResult[]; ndr: ReviewActionResult[] | undefined }> { txids = normalizePostTxids(txids, 'txids', true) const swr: SendWithResult[] = [] const ndr: ReviewActionResult[] | undefined = undefined @@ -316,7 +320,11 @@ export async function shareReqsWithWorld ( ? await getReqDetailsForDelayedShare(storage, txids) : await storage.getReqsAndBeefToShareWithWorld(txids, []) - normalizePostTxids(r.details.map(detail => detail.txid), 'details', true) + normalizePostTxids( + r.details.map(detail => detail.txid), + 'details', + true + ) const readyToSendReqs: EntityProvenTxReq[] = [] if (!classifyReqDetails(r.details, swr, readyToSendReqs)) return { swr, ndr } @@ -356,12 +364,14 @@ interface ReqTxStatus { tx: TransactionStatus } -function determineReqTxStatus (params: Pick): { +function determineReqTxStatus(params: Pick): { status: ReqTxStatus postStatus: ReqTxStatus | undefined } { if (params.isNoSend && !params.isSendWith) return { status: { req: 'nosend', tx: 'nosend' }, postStatus: undefined } - if (!params.isNoSend && params.isDelayed) { return { status: { req: 'unsent', tx: 'unprocessed' }, postStatus: undefined } } + if (!params.isNoSend && params.isDelayed) { + return { status: { req: 'unsent', tx: 'unprocessed' }, postStatus: undefined } + } if (!params.isNoSend && !params.isDelayed) { return { status: { req: 'unprocessed', tx: 'unprocessed' }, @@ -371,7 +381,7 @@ function determineReqTxStatus (params: Pick }> + outputUpdates: Array<{ id: number; update: Partial }> transactionUpdate: Partial postStatus?: ReqTxStatus } -function parseProcessActionTransaction (params: StorageProcessActionArgs): { +function parseProcessActionTransaction(params: StorageProcessActionArgs): { reference: string txid: string rawTx: number[] @@ -449,7 +459,7 @@ function parseProcessActionTransaction (params: StorageProcessActionArgs): { return { reference, txid, rawTx, tx } } -async function validateNoSendExpiryRelease ( +async function validateNoSendExpiryRelease( storage: StorageProvider, auth: AuthId, params: StorageProcessActionArgs, @@ -471,7 +481,7 @@ async function validateNoSendExpiryRelease ( if (expired) throw new WERR_INVALID_OPERATION('BRC-177 protected action has expired') } -function validatePlannedTransaction (transaction: TableTransaction): void { +function validatePlannedTransaction(transaction: TableTransaction): void { if (!transaction.isOutgoing) throw new WERR_INVALID_OPERATION('isOutgoing is not true') if (transaction.inputBEEF == null) throw new WERR_INVALID_OPERATION() if (transaction.status !== 'unsigned' && transaction.status !== 'unprocessed') { @@ -479,7 +489,7 @@ function validatePlannedTransaction (transaction: TableTransaction): void { } } -function validateCommissionOutput ( +function validateCommissionOutput( storage: StorageProvider, tx: BsvTransaction, commissionRows: TableCommission[] @@ -496,7 +506,7 @@ function validateCommissionOutput ( } } -async function validateCommitNewTxToStorageArgs ( +async function validateCommitNewTxToStorageArgs( storage: StorageProvider, auth: AuthId, params: StorageProcessActionArgs @@ -590,7 +600,7 @@ export interface CommitNewTxResults { log?: string } -async function commitNewTxToStorage ( +async function commitNewTxToStorage( storage: StorageProvider, userId: number, vargs: ValidCommitNewTxToStorageArgs diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/proofWork.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/proofWork.test.ts new file mode 100644 index 000000000..a22916018 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/proofWork.test.ts @@ -0,0 +1,85 @@ +import type { StorageProvider } from '../StorageProvider' +import { findProofRecords, mapProofWork } from './proofWork' + +test('bounds proof concurrency and drains already-started I/O before rejecting', async () => { + let release!: () => void + const gate = new Promise(resolve => { + release = resolve + }) + let active = 0 + let started = 0 + let peak = 0 + let settled = false + const failure = new Error('proof lookup failed') + const pending = mapProofWork( + Array.from({ length: 20 }, (_, index) => index), + async index => { + active++ + started++ + peak = Math.max(peak, active) + try { + if (index === 0) throw failure + await gate + return index + } finally { + active-- + } + } + ) + const rejected = expect(pending).rejects.toBe(failure) + void pending.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + await Promise.resolve() + await Promise.resolve() + expect(started).toBe(8) + expect(peak).toBeLessThanOrEqual(8) + expect(settled).toBe(false) + release() + await rejected + expect(active).toBe(0) + expect(started).toBe(8) +}) + +test('deduplicates and bounds large proof lookups before reaching SQL bindings', async () => { + let active = 0 + let peak = 0 + const findProvenTxs = jest.fn(async () => { + active++ + peak = Math.max(peak, active) + await Promise.resolve() + active-- + return [] + }) + const txids = Array.from({ length: 1201 }, (_, i) => i.toString(16).padStart(64, '0')) + await expect( + findProofRecords({ findProvenTxs } as unknown as StorageProvider, [...txids, ...txids]) + ).resolves.toEqual([]) + expect(findProvenTxs).toHaveBeenCalledTimes(5) + const requests = findProvenTxs.mock.calls as unknown as [{ txids: string[] }][] + expect(requests.every(([args]) => args.txids.length <= 250)).toBe(true) + expect(requests.flatMap(([args]) => args.txids)).toEqual(txids) + expect(peak).toBe(1) + expect(active).toBe(0) +}) + +test('keeps proof results in input order across multiple bounded worker batches', async () => { + let active = 0 + let peak = 0 + const items = Array.from({ length: 21 }, (_, index) => index) + const results = await mapProofWork(items, async index => { + active++ + peak = Math.max(peak, active) + await new Promise(resolve => setTimeout(resolve, 8 - (index % 8))) + active-- + return index * 2 + }) + expect(results).toEqual(items.map(index => index * 2)) + expect(peak).toBe(8) + expect(active).toBe(0) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/proofWork.ts b/packages/wallet/wallet-toolbox/src/storage/methods/proofWork.ts new file mode 100644 index 000000000..e351a24ae --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/proofWork.ts @@ -0,0 +1,48 @@ +import { runInSeries } from '../../utility/runInSeries' +import type { StorageProvider } from '../StorageProvider' +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' +import type { TableProvenTx } from '../schema/tables' + +function* proofBatches(txids: string[]) { + for (let offset = 0; offset < txids.length; offset += 250) yield txids.slice(offset, offset + 250) +} + +/** Bound SQL parameters even for legacy callers supplying a large proof page. */ +export async function findProofRecords( + storage: StorageProvider, + txids: string[], + trx?: TrxToken +): Promise { + const unique = [...new Set(txids)] + const records: TableProvenTx[] = [] + // Consume one SQL batch at a time; callers may share a transaction connection. + await runInSeries(proofBatches(unique), async batch => { + records.push(...(await storage.findProvenTxs({ partial: {}, txids: batch, trx }))) + }) + return records +} + +/** All started work drains before a failure escapes; at most eight checks run. */ +export async function mapProofWork(items: T[], work: (item: T) => Promise): Promise { + const results: R[] = [] + let next = 0 + let failed = false + function* pendingIndexes() { + while (!failed && next < items.length) yield next++ + } + const workers = Array.from({ length: Math.min(items.length, 8) }, async () => { + // Each worker consumes serially; all eight share the same admission cursor. + await runInSeries(pendingIndexes(), async index => { + if (failed) return + try { + results[index] = await work(items[index]) + } catch (error) { + failed = true + throw error + } + }) + }) + const settled = await Promise.allSettled(workers) + for (const result of settled) if (result.status === 'rejected') throw result.reason + return results +} diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/refreshSyncProof.ts b/packages/wallet/wallet-toolbox/src/storage/methods/refreshSyncProof.ts new file mode 100644 index 000000000..59dc03966 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/refreshSyncProof.ts @@ -0,0 +1,49 @@ +import { WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +import type { GetMerklePathResult } from '../../sdk/WalletServices.interfaces' +import { doubleSha256BE } from '../../utility/utilityHelpers' +import { asString } from '../../utility/utilityHelpers.noBuffer' +import { validateSyncProof, markSyncProofReconciled } from './validateSyncProof' +import type { SyncProofValidationStorage } from './validateSyncProof' +import type { TableProvenTx } from '../schema/tables/TableProvenTx' + +/** Obtain replacement metadata; never alter transaction bytes, IDs or timestamps. */ +export async function refreshSyncProof( + storage: SyncProofValidationStorage, + candidate: TableProvenTx +): Promise { + const services = storage.getServices() + let current: TableProvenTx | undefined + const validate = async ({ merklePath }: GetMerklePathResult): Promise => { + if (merklePath == null) throw new Error('Current proof unavailable') + const header = await services.getHeaderForHeight(merklePath.blockHeight) + const replacement = { + ...candidate, + height: merklePath.blockHeight, + index: merklePath.path[0]?.find(leaf => leaf.txid === true && leaf.hash === candidate.txid)?.offset ?? -1, + merklePath: merklePath.toBinary(), + merkleRoot: merklePath.computeRoot(candidate.txid), + blockHash: asString(doubleSha256BE(header)) + } + // Recheck raw bytes, membership, root and active header before authorizing + // any global proof replacement. A provider's success response is insufficient. + await validateSyncProof(storage, replacement) + markSyncProofReconciled(replacement) + current = replacement + } + try { + if (services.getValidatedMerklePath != null) { + await services.getValidatedMerklePath(candidate.txid, validate) + } else { + // Custom services remain compatible; one bounded lookup, fully validated. + await validate(await services.getMerklePath(candidate.txid)) + } + if (current != null) return current + } catch { + // Report a recoverable sync error, without leaking provider response data. + } + throw new WERR_INVALID_PARAMETER( + 'provenTx', + 'a server-verified proof. Merkle root is not active at the recorded height and a current proof could not be verified. ' + + 'Ask the source provider to reconcile this transaction, then resume synchronization; saved wallet data is unchanged.' + ) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/repairBeefProofs.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/repairBeefProofs.test.ts new file mode 100644 index 000000000..c1a0c61e8 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/repairBeefProofs.test.ts @@ -0,0 +1,535 @@ +import 'fake-indexeddb/auto' +import { randomUUID } from 'node:crypto' +import { once } from 'node:events' +import { StorageServer } from '../remoting/StorageServer' +import { StorageClient } from '../remoting/StorageClient' +import { KnexSessionManager } from '../remoting/KnexSessionManager' +import { Beef, MerklePath, PrivateKey, Script, Transaction, Validation } from '@bsv/sdk' +import { StorageIdb } from '../StorageIdb' +import { StorageProvider } from '../StorageProvider' +import { WalletStorageManager } from '../WalletStorageManager' +import { repairBeefProofs } from './repairBeefProofs' +import { validateSyncProof } from './validateSyncProof' +import { WERR_INVALID_MERKLE_ROOT } from '../../sdk/WERR_errors' +import { WalletErrorFromJson } from '../../sdk/WalletErrorFromJson' +import { toBinaryBaseBlockHeader } from '../../services/Services' +import { doubleSha256BE } from '../../utility/utilityHelpers' +import { asString } from '../../utility/utilityHelpers.noBuffer' +import type { WalletServices } from '../../sdk/WalletServices.interfaces' +import { _tu } from '../../../test/utils/TestUtilsWalletStorage' +import { managedChangeOutputFields } from './managedChange' +import { EntityProvenTxReq } from '../schema/entities/EntityProvenTxReq' +import type { RequestSyncChunkArgs, SyncChunk, WalletStorageSyncReader } from '../../sdk/WalletStorage.interfaces' + +function fixture() { + const tx = new Transaction() + tx.addOutput({ satoshis: 5000, lockingScript: Script.fromHex('51') }) + const txid = tx.id('hex') + const old = new MerklePath(100, [[{ offset: 0, hash: txid, txid: true }]]) + const current = new MerklePath(101, [ + [ + { offset: 0, hash: txid, txid: true }, + { offset: 1, hash: 'ab'.repeat(32) } + ] + ]) + const root = current.computeRoot(txid) + const header = toBinaryBaseBlockHeader({ + version: 1, + previousHash: '00'.repeat(32), + merkleRoot: root, + time: 1, + bits: 0, + nonce: 0 + }) + const record = { + provenTxId: 0, + txid, + rawTx: tx.toBinary(), + merklePath: old.toBinary(), + height: 100, + index: 0, + merkleRoot: old.computeRoot(txid), + blockHash: '11'.repeat(32), + created_at: new Date(0), + updated_at: new Date(1) + } + const getMerklePath = jest.fn(async () => ({ merklePath: current })) + const isValidRootForHeight = jest.fn( + async (candidate: string, height: number) => candidate === root && height === 101 + ) + const getHeaderForHeight = jest.fn(async () => [...header]) + const services = { + getMerklePath, + getHeaderForHeight, + getChainTracker: async () => ({ isValidRootForHeight }) + } as unknown as WalletServices + const beef = new Beef() + beef.mergeRawTx(record.rawTx) + beef.mergeBump(old) + return { beef, record, services, current, root, header, getMerklePath, isValidRootForHeight, getHeaderForHeight } +} + +async function seedFunding(storage: StorageProvider, f: ReturnType, identityKey: string) { + const { user } = await storage.findOrInsertUser(identityKey) + const basket = await storage.findOrInsertOutputBasket(user.userId, 'default') + await storage.updateOutputBasket(basket.basketId, { numberOfDesiredUTXOs: 0, minimumDesiredUTXOValue: 1 }) + const { tx } = await _tu.insertTestTransaction(storage, user, false, { + txid: f.record.txid, + rawTx: f.record.rawTx, + provenTxId: f.record.provenTxId, + status: 'completed' + }) + const output = await _tu.insertTestOutput(storage, tx, 0, 5000, basket, true, { + ...managedChangeOutputFields, + txid: f.record.txid, + lockingScript: [0x51], + scriptLength: 1, + derivationPrefix: 'dGVzdA==', + derivationSuffix: 'dGVzdA==' + }) + const args = Validation.validateCreateActionArgs({ + description: 'selected stale proof recovery', + outputs: [{ satoshis: 1000, lockingScript: '51', outputDescription: 'proof recovery output' }], + options: { noSend: true, returnTXIDOnly: false, randomizeOutputs: false } + }) + return { user, output, args } +} + +describe.each(['IndexedDB', 'SQLite'])('canonical BEEF recovery on %s', backend => { + let storage: StorageProvider + let cleanup: () => Promise + beforeEach(async () => { + if (backend === 'IndexedDB') { + const idb = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + idb.dbName = `proof-recovery-${randomUUID()}` + await idb.migrate('proof recovery', PrivateKey.fromRandom().toPublicKey().toString()) + await idb.makeAvailable() + storage = idb + cleanup = async () => { + await idb.destroy() + await idb.dropAllData() + } + } else { + const ctx = await _tu.createLegacyWalletSQLiteCopy('proof-recovery') + storage = ctx.activeStorage + cleanup = async () => { + await ctx.wallet.destroy() + } + } + }) + afterEach(async () => { + jest.restoreAllMocks() + await cleanup() + }) + + test('repairs stale selected or embedded ancestry, persists atomically and reuses the correction', async () => { + const f = fixture() + storage.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + const original = f.beef.toBinary() + const result = await repairBeefProofs(storage, f.beef) + expect(result.findTxid(f.record.txid)?.rawTx).toEqual(f.record.rawTx) + expect(result.bumps.map(path => path.toBinary())).toEqual([f.current.toBinary()]) + expect(f.beef.toBinary()).toEqual(original) + expect(f.getMerklePath).toHaveBeenCalledTimes(1) + const stored = (await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0] + expect(stored).toMatchObject({ height: 101, blockHash: asString(doubleSha256BE(f.header)), merkleRoot: f.root }) + expect(stored.updated_at.getTime()).toBeGreaterThan(f.record.updated_at.getTime()) + const repeated = await repairBeefProofs(storage, f.beef) + expect(repeated.toBinary()).toEqual(result.toBinary()) + expect(f.getMerklePath).toHaveBeenCalledTimes(1) + }) + + test.each(['absent', 'orphan', 'wrong header', 'throw'])( + 'fails with typed remote context for %s canonical evidence', + async failure => { + const f = fixture() + storage.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + const before = f.beef.toBinary() + if (failure === 'absent') f.getMerklePath.mockResolvedValue({ merklePath: undefined! }) + if (failure === 'orphan') f.isValidRootForHeight.mockResolvedValue(false) + if (failure === 'wrong header') f.getHeaderForHeight.mockResolvedValue(Array(80).fill(0)) + if (failure === 'throw') f.getMerklePath.mockRejectedValue(new Error('unavailable')) + const error = await repairBeefProofs(storage, f.beef).catch(error => error) + expect(error).toBeInstanceOf(WERR_INVALID_MERKLE_ROOT) + expect(WalletErrorFromJson(JSON.parse(error.toJson()))).toMatchObject({ txid: f.record.txid, blockHeight: 100 }) + expect(f.beef.toBinary()).toEqual(before) + expect(f.getMerklePath).toHaveBeenCalledTimes(1) + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0].height).toBe(100) + } + ) + + test('refuses unvalidated replacements and does not overwrite a concurrently changed proof', async () => { + const f = fixture() + storage.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + const replacement = { + ...f.record, + height: 101, + merklePath: f.current.toBinary(), + merkleRoot: f.root, + blockHash: asString(doubleSha256BE(f.header)) + } + await expect(storage.compareAndSetProvenTxProof(f.record, replacement)).rejects.toThrow( + 'requires active-chain validation' + ) + await validateSyncProof(storage, replacement) + await storage.updateProvenTx(f.record.provenTxId, { height: 102 }) + expect(await storage.compareAndSetProvenTxProof(f.record, replacement)).toBe(false) + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0].height).toBe(102) + }) + + test.each([false, true])( + 'prepared sync detaches payloads and fences concurrent proof changes (%s)', + async changed => { + const f = fixture() + storage.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + const identityKey = PrivateKey.fromRandom().toPublicKey().toString() + const source = PrivateKey.fromRandom().toPublicKey().toString() + const checkpoint = await storage.getSyncCheckpoint({ identityKey }, source, 'prepared fixture') + const args: RequestSyncChunkArgs = { + ...checkpoint, + identityKey, + fromStorageIdentityKey: source, + toStorageIdentityKey: storage.getSettings().storageIdentityKey, + maxItems: 64, + maxRoughSize: 1000000, + requireMatchingCheckpoint: true, + includeNextCheckpoint: true + } + const replacement = { + ...f.record, + height: 101, + merklePath: f.current.toBinary(), + merkleRoot: f.root, + blockHash: asString(doubleSha256BE(f.header)) + } + const chunk: SyncChunk = { + fromStorageIdentityKey: source, + toStorageIdentityKey: args.toStorageIdentityKey, + userIdentityKey: identityKey, + provenTxs: [replacement] + } + const apply = await storage.prepareSyncChunk(args, chunk) + if (changed) { + await storage.updateProvenTx(f.record.provenTxId, { height: 102 }) + await expect(apply()).rejects.toThrow('Proof changed during sync preparation') + expect(await storage.getSyncCheckpoint({ identityKey }, source, 'prepared fixture')).toEqual(checkpoint) + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0].height).toBe(102) + } else { + replacement.rawTx[0] ^= 1 + replacement.merklePath[0] ^= 1 + args.offsets[0].offset = 9999 + const committed = await apply() + expect(committed.nextCheckpoint?.offsets[0].offset).toBe(1) + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0]).toMatchObject({ + height: 101, + merkleRoot: f.root + }) + } + await expect(apply()).rejects.toThrow('already consumed') + } + ) + + test('sync proof I/O yields the manager queue and cancellation discards the prepared page', async () => { + const f = fixture() + storage.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + const identityKey = PrivateKey.fromRandom().toPublicKey().toString() + const manager = new WalletStorageManager(identityKey, storage) + await manager.makeAvailable() + const sourceSettings = { + ...storage.getSettings(), + storageIdentityKey: PrivateKey.fromRandom().toPublicKey().toString() + } + const reader = { + makeAvailable: async () => sourceSettings, + getSettings: () => sourceSettings, + getSyncChunk: async (args: RequestSyncChunkArgs): Promise => ({ + fromStorageIdentityKey: sourceSettings.storageIdentityKey, + toStorageIdentityKey: args.toStorageIdentityKey, + userIdentityKey: identityKey, + provenTxs: [ + { + ...f.record, + height: 101, + merklePath: f.current.toBinary(), + merkleRoot: f.root, + blockHash: asString(doubleSha256BE(f.header)) + } + ] + }) + } as WalletStorageSyncReader + let entered!: () => void + let release!: () => void + const started = new Promise(resolve => { + entered = resolve + }) + const gate = new Promise(resolve => { + release = resolve + }) + f.getHeaderForHeight.mockImplementation(async () => { + entered() + await gate + return f.header + }) + const controller = new AbortController() + const sync = manager.syncFromReaderResumable(identityKey, reader, { signal: controller.signal }) + await started + try { + const userId = await manager.getUserId() + await manager.runAsWriter(async () => await storage.findOrInsertTxLabel(userId, 'during proof lookup')) + expect(await manager.runAsReader(async () => await storage.countTxLabels({ partial: { userId } }))).toBe(1) + controller.abort() + } finally { + release() + } + expect(await sync).toMatchObject({ mode: 'paged', status: 'cancelled', pages: 0 }) + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0].height).toBe(100) + }, 10000) + + test.each(['direct', 'header', 'height'])( + 'monitor %s recovery cannot overwrite a concurrent proof correction', + async mode => { + const f = fixture() + const manager = new WalletStorageManager(PrivateKey.fromRandom().toPublicKey().toString(), storage) + await manager.makeAvailable() + manager.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + let entered!: () => void + let release!: () => void + const started = new Promise(resolve => { + entered = resolve + }) + const paused = new Promise(resolve => { + release = resolve + }) + f.getMerklePath.mockImplementation(async () => { + entered() + await paused + return { merklePath: f.current } + }) + const recovering = + mode === 'direct' + ? manager.reproveProven(f.record) + : mode === 'header' + ? manager.reproveHeader(f.record.blockHash) + : manager.reproveHeightMerkleRoot(f.record.height, f.record.merkleRoot) + await started + try { + await manager.runAsWriter(async () => await storage.updateProvenTx(f.record.provenTxId, { height: 102 })) + } finally { + release() + } + const result = await recovering + expect(result.updated == null || (Array.isArray(result.updated) && result.updated.length === 0)).toBe(true) + expect(result.log).toContain('changed concurrently') + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0].height).toBe(102) + }, + 10000 + ) + + test('monitor recovery fences primary replacement before committing a late proof', async () => { + const f = fixture() + const manager = new WalletStorageManager(PrivateKey.fromRandom().toPublicKey().toString(), storage) + const next = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + next.dbName = `reproof-next-${randomUUID()}` + await next.migrate('reproof replacement', PrivateKey.fromRandom().toPublicKey().toString()) + await manager.makeAvailable() + await manager.addWalletStorageProvider(next) + await manager.setActive(storage.getSettings().storageIdentityKey) + manager.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + let entered!: () => void + let release!: () => void + const started = new Promise(resolve => { + entered = resolve + }) + const paused = new Promise(resolve => { + release = resolve + }) + f.getMerklePath.mockImplementation(async () => { + entered() + await paused + return { merklePath: f.current } + }) + const recovering = manager.reproveHeader(f.record.blockHash) + const rejected = expect(recovering).rejects.toThrow('Proof destination changed') + await started + try { + await manager.setActive(next.getSettings().storageIdentityKey) + } finally { + release() + } + try { + await rejected + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0].height).toBe(100) + expect(await next.findProvenTxs({ partial: { txid: f.record.txid } })).toHaveLength(0) + } finally { + await next.destroy() + await next.dropAllData() + } + }, 10000) + + test('repairs embedded ancestry absent from storage and preserves txid-only references', async () => { + const f = fixture() + storage.setServices(f.services) + const reference = 'ac'.repeat(32) + f.beef.mergeTxidOnly(reference) + const repaired = await repairBeefProofs(storage, f.beef) + expect(repaired.findTxid(reference)?.isTxidOnly).toBe(true) + expect(repaired.findTxid(f.record.txid)?.rawTx).toEqual(f.record.rawTx) + expect(repaired.bumps[0].computeRoot(f.record.txid)).toBe(f.root) + expect(await storage.countProvenTxs({ partial: { txid: f.record.txid } })).toBe(0) + }) + + test('reuses a concurrent canonical database correction without another provider request', async () => { + const f = fixture() + storage.setServices(f.services) + await storage.insertProvenTx({ + ...f.record, + height: 101, + merklePath: f.current.toBinary(), + merkleRoot: f.root, + blockHash: asString(doubleSha256BE(f.header)) + }) + const persist = jest.spyOn(storage, 'compareAndSetProvenTxProof') + const repaired = await repairBeefProofs(storage, f.beef) + expect(repaired.bumps[0].computeRoot(f.record.txid)).toBe(f.root) + expect(f.getMerklePath).not.toHaveBeenCalled() + expect(persist).not.toHaveBeenCalled() + }) + + test('a provider without atomic proof replacement can repair outgoing BEEF without changing shared storage', async () => { + const f = fixture() + storage.setServices(f.services) + await storage.insertProvenTx(f.record) + jest + .spyOn(storage, 'compareAndSetProvenTxProof') + .mockImplementation(StorageProvider.prototype.compareAndSetProvenTxProof.bind(storage)) + const repaired = await repairBeefProofs(storage, f.beef) + expect(repaired.bumps[0].computeRoot(f.record.txid)).toBe(f.root) + expect((await storage.findProvenTxs({ partial: { txid: f.record.txid } }))[0].height).toBe(100) + }) + + test('keeps valid proof assembly free of copies and storage/provider lookups', async () => { + const f = fixture() + storage.setServices(f.services) + f.isValidRootForHeight.mockResolvedValue(true) + const find = jest.spyOn(storage, 'findProvenTxs') + expect(await repairBeefProofs(storage, f.beef)).toBe(f.beef) + expect(find).not.toHaveBeenCalled() + expect(f.getMerklePath).not.toHaveBeenCalled() + expect(f.isValidRootForHeight).toHaveBeenCalledTimes(1) + }) + + test.each([false, true])( + 'validates selected change before returning to the signer (recovery unavailable=%s)', + async unavailable => { + const f = fixture() + storage.setServices(f.services) + f.record.provenTxId = await storage.insertProvenTx(f.record) + const { user, output, args } = await seedFunding(storage, f, PrivateKey.fromRandom().toPublicKey().toString()) + if (unavailable) { + f.getMerklePath.mockResolvedValue({ merklePath: undefined! }) + await expect( + storage.createAction({ identityKey: user.identityKey, userId: user.userId }, args) + ).rejects.toBeInstanceOf(WERR_INVALID_MERKLE_ROOT) + const retained = (await storage.findOutputs({ partial: { outputId: output.outputId } }))[0] + expect(retained.spendable).toBe(true) + expect(retained.spentBy).toBeUndefined() + expect(await storage.findTransactions({ partial: { userId: user.userId }, status: ['failed'] })).toHaveLength(1) + f.getMerklePath.mockResolvedValue({ merklePath: f.current }) + } + const created = await storage.createAction({ identityKey: user.identityKey, userId: user.userId }, args) + expect(Beef.fromBinary(created.inputBeef!).bumps.map(bump => bump.computeRoot())).toEqual([f.root]) + expect(created.inputs).toHaveLength(1) + expect(created.inputs[0].sourceTxid).toBe(f.record.txid) + } + ) + + test.each([false, true])( + 'checks the final rebuilt send bundle before broadcast (recovery unavailable=%s)', + async unavailable => { + const f = fixture() + const stop = new Error('stop at controlled broadcaster') + const postBeef = jest.fn(async (_beef: Beef) => { + throw stop + }) + storage.setServices({ ...f.services, postBeef } as WalletServices) + f.record.provenTxId = await storage.insertProvenTx(f.record) + const tx = new Transaction() + tx.addInput({ sourceTXID: f.record.txid, sourceOutputIndex: 0, unlockingScript: Script.fromHex('') }) + tx.addOutput({ satoshis: 4999, lockingScript: Script.fromHex('51') }) + const req = EntityProvenTxReq.fromTxid(tx.id('hex'), tx.toBinary(), f.beef.toBinary()) + req.notify.transactionIds = [1] + if (unavailable) f.getMerklePath.mockResolvedValue({ merklePath: undefined! }) + const result = storage.attemptToPostReqsToNetwork([req]) + if (unavailable) { + await expect(result).rejects.toBeInstanceOf(WERR_INVALID_MERKLE_ROOT) + expect(postBeef).not.toHaveBeenCalled() + expect(req.attempts).toBe(0) + } else { + await expect(result).rejects.toBe(stop) + expect(postBeef).toHaveBeenCalledTimes(1) + expect(postBeef.mock.calls[0][0].bumps.map(bump => bump.computeRoot())).toEqual([f.root]) + } + } + ) +}) + +test('remote selected-change recovery binds the authenticated wallet and preserves typed errors', async () => { + const ctx = await _tu.createLegacyWalletSQLiteCopy('remote-proof-recovery') + const server = new StorageServer(ctx.activeStorage, { + port: 0, + wallet: ctx.wallet, + monetize: false, + logRpcRequests: false, + sessionManager: new KnexSessionManager(ctx.activeStorage.knex), + adminIdentityKeys: [], + calculateRequestPrice: async () => 0 + }) + server.start() + if (!server.server.listening) await once(server.server, 'listening') + const address = server.server.address() + if (address == null || typeof address === 'string') throw new Error('server did not bind') + const endpointUrl = `http://localhost:${address.port}` + const client = await _tu.createTestWalletWithStorageClient({ + rootKeyHex: ctx.rootKey.toHex(), + endpointUrl, + chain: ctx.chain + }) + const stranger = await _tu.createTestWalletWithStorageClient({ + rootKeyHex: '2'.repeat(64), + endpointUrl, + chain: ctx.chain + }) + try { + const f = fixture() + ctx.activeStorage.setServices(f.services) + for (const output of await ctx.activeStorage.findOutputs({ partial: { userId: ctx.userId }, noScript: true })) { + if (output.spendable) await ctx.activeStorage.updateOutput(output.outputId, { spendable: false }) + } + f.record.provenTxId = await ctx.activeStorage.insertProvenTx(f.record) + const { args } = await seedFunding(ctx.activeStorage, f, ctx.identityKey) + const forgedAuth = { identityKey: ctx.identityKey, userId: ctx.userId, isActive: true } + await expect((stranger.storage.getActive() as StorageClient).createAction(forgedAuth, args)).rejects.toThrow() + expect(f.getMerklePath).not.toHaveBeenCalled() + f.getMerklePath.mockResolvedValue({ merklePath: undefined! }) + await expect((client.storage.getActive() as StorageClient).createAction(forgedAuth, args)).rejects.toMatchObject({ + name: 'WERR_INVALID_MERKLE_ROOT', + txid: f.record.txid, + blockHeight: 100 + }) + f.getMerklePath.mockResolvedValue({ merklePath: f.current }) + const created = await (client.storage.getActive() as StorageClient).createAction(forgedAuth, args) + expect(Beef.fromBinary(created.inputBeef!).bumps.map(bump => bump.computeRoot())).toEqual([f.root]) + } finally { + await stranger.wallet.destroy() + await client.wallet.destroy() + await server.close() + await ctx.wallet.destroy() + } +}, 30000) diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/repairBeefProofs.ts b/packages/wallet/wallet-toolbox/src/storage/methods/repairBeefProofs.ts new file mode 100644 index 000000000..37edfe8b6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/methods/repairBeefProofs.ts @@ -0,0 +1,99 @@ +import { runInSeries } from '../../utility/runInSeries' +import { findProofRecords, mapProofWork } from './proofWork' +import { Beef, MerklePath } from '@bsv/sdk' +import { WERR_INVALID_MERKLE_ROOT } from '../../sdk/WERR_errors' +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' +import type { StorageProvider } from '../StorageProvider' +import type { TableProvenTx } from '../schema/tables' +import { refreshSyncProof } from './refreshSyncProof' +import { validateSyncProof } from './validateSyncProof' + +/** + * Validate the assembled graph, including cached and embedded ancestry. Only a + * stale root triggers canonical recovery. Never change transaction bytes or + * treat provider absence as a successful proof. The common valid path neither + * copies the graph nor reads proof records from storage. + */ +export async function repairBeefProofs(storage: StorageProvider, beef: Beef, trx?: TrxToken): Promise { + if (beef.bumps.length === 0) return beef + const services = storage.getServices() + const tracker = await services.getChainTracker() + const roots = new Map() + for (const bump of beef.bumps) { + const root = bump.computeRoot() + roots.set(`${bump.blockHeight}:${root}`, { root, height: bump.blockHeight }) + } + const checks = await mapProofWork( + [...roots.entries()], + async ([key, { root, height }]) => [key, (await tracker.isValidRootForHeight(root, height)) === true] as const + ) + const valid = new Map(checks) + const stale = new Set(beef.bumps.filter(bump => !valid.get(`${bump.blockHeight}:${bump.computeRoot()}`))) + if (stale.size === 0) return beef + + const affected = beef.txs.filter(tx => tx.bumpIndex != null && stale.has(beef.bumps[tx.bumpIndex])) + const records = await findProofRecords( + storage, + affected.map(tx => tx.txid), + trx + ) + const existing = new Map(records.map(record => [record.txid, record])) + const replacements = await mapProofWork(affected, async tx => { + const oldPath = beef.bumps[tx.bumpIndex!] + const record = existing.get(tx.txid) + if (record != null) { + try { + // Another request may already have repaired the shared proof row. + await validateSyncProof(storage, record) + return { proof: record } + } catch { + // Recovery still requires full validation of fresh canonical evidence. + } + } + const candidate: TableProvenTx = { + provenTxId: record?.provenTxId ?? 0, + created_at: record?.created_at ?? new Date(0), + updated_at: record?.updated_at ?? new Date(0), + txid: tx.txid, + rawTx: tx.rawTx ?? [], + height: oldPath.blockHeight, + merklePath: oldPath.toBinary(), + merkleRoot: oldPath.computeRoot(tx.txid), + blockHash: record?.blockHash ?? '00'.repeat(32), + index: oldPath.path[0].find(leaf => leaf.hash === tx.txid)?.offset ?? -1 + } + try { + return { proof: await refreshSyncProof(storage, candidate), expected: record } + } catch { + const error = new WERR_INVALID_MERKLE_ROOT(candidate.blockHash, candidate.height, candidate.merkleRoot, tx.txid) + error.message += + ' A current canonical proof could not be verified. Retry after the proof provider recovers; no transaction was broadcast by proof recovery.' + throw error + } + }) + + // Construct independently: failed recovery must not leave a half-edited BEEF. + const repaired = new Beef(beef.version) + for (const tx of beef.txs) { + const bytes = tx.rawTxUint8Array + if (bytes == null) repaired.mergeTxidOnly(tx.txid) + else repaired.mergeRawTx(bytes) + } + for (const bump of beef.bumps) if (!stale.has(bump)) repaired.mergeBump(bump) + for (const { proof } of replacements) repaired.mergeBump(MerklePath.fromBinary(proof.merklePath)) + + // Persistence is optional for custom providers. Built-ins compare all proof + // authority fields atomically, so delayed I/O cannot overwrite a newer proof. + if (replacements.some(result => result.expected != null)) { + await storage.transaction(async trx => { + let changed = false + // Preserve transaction write order and finish each compare-and-set before the next. + await runInSeries(replacements, async ({ proof, expected }) => { + if (expected != null) changed = (await storage.compareAndSetProvenTxProof(expected, proof, trx)) || changed + }) + const extension = storage as StorageProvider & { invalidatePreparedBeefs?: (trx?: TrxToken) => Promise } + if (changed) await extension.invalidatePreparedBeefs?.(trx) + }, trx) + } + return repaired +} diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.test.ts b/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.test.ts index 44ea1a162..5532f4edf 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.test.ts @@ -161,3 +161,47 @@ test('exhausts providers once without accepting an invalid proof or partially ch expect(page).toEqual(before) expect(() => assertSyncProofReplacementAuthorized(page[0])).toThrow() }) + +test('validates packed proof bytes without expanding or changing them', async () => { + const f = fixture() + const rawTx = new Uint8Array(f.candidate.rawTx) + const merklePath = new Uint8Array(f.currentPath.toBinary()) + const candidate = { + ...f.candidate, + height: 101, + rawTx: rawTx as unknown as number[], + merklePath: merklePath as unknown as number[], + merkleRoot: f.currentRoot, + blockHash: asString(doubleSha256BE(f.currentHeader)), + index: 0 + } + await validateSyncProof(f.storage, candidate) + expect(candidate.rawTx).toBe(rawTx) + expect(candidate.merklePath).toBe(merklePath) + expect(() => assertSyncProofReplacementAuthorized(candidate)).not.toThrow() +}) + +describe.each([ + ['rawTx', 'raw transaction is required'], + ['merklePath', 'Merkle path is required'] +] as const)('%s input admission', (field, reason) => { + test.each([ + ['empty number array', []], + ['empty packed array', new Uint8Array()], + ['string', '01'], + ['array-like object', { 0: 1, length: 1 }], + ['missing value', undefined], + ['null', null] + ])('rejects %s before service access or proof authorization', async (_description, bytes) => { + const f = fixture() + const services = jest.spyOn(f.storage, 'getServices') + const candidate = { ...f.candidate, [field]: bytes as unknown as number[] } + const original = structuredClone(candidate) + await expect(validateSyncProof(f.storage, candidate)).rejects.toThrow(reason) + expect(services).not.toHaveBeenCalled() + expect(candidate).toEqual(original) + expect(() => assertSyncProofReplacementAuthorized(candidate)).toThrow( + 'replacement requires active-chain validation' + ) + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.ts b/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.ts index ff9d58676..5032eda3d 100644 --- a/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.ts +++ b/packages/wallet/wallet-toolbox/src/storage/methods/validateSyncProof.ts @@ -85,6 +85,22 @@ export function assertSyncProofReplacementAuthorized(candidate: TableProvenTx): invalidSyncProof('replacement requires active-chain validation') } +/** Restrict recovery updates to validated proof authority and a discoverable timestamp. */ +export function recoveredProofUpdate(expected: TableProvenTx, replacement: TableProvenTx): Partial { + assertSyncProofReplacementAuthorized(replacement) + if (expected.txid !== replacement.txid || !equalBytes(expected.rawTx, replacement.rawTx)) { + invalidSyncProof('recovery cannot change transaction identity or bytes') + } + return { + height: replacement.height, + index: replacement.index, + merklePath: replacement.merklePath, + merkleRoot: replacement.merkleRoot, + blockHash: replacement.blockHash, + updated_at: new Date(Math.max(Date.now(), expected.updated_at.getTime() + 1)) + } +} + /** * Validate proof authority before an RPC proof is admitted or an in-process * backup/conflict proof replaces an existing global row. Network-backed checks @@ -98,10 +114,13 @@ export async function validateSyncProof(storage: SyncProofValidationStorage, can if (!Number.isSafeInteger(candidate.index) || candidate.index < 0) { invalidSyncProof('index must be a non-negative safe integer') } - if (!Array.isArray(candidate.rawTx) || candidate.rawTx.length === 0) { + if (!(candidate.rawTx instanceof Uint8Array || Array.isArray(candidate.rawTx)) || candidate.rawTx.length === 0) { invalidSyncProof('raw transaction is required') } - if (!Array.isArray(candidate.merklePath) || candidate.merklePath.length === 0) { + if ( + !(candidate.merklePath instanceof Uint8Array || Array.isArray(candidate.merklePath)) || + candidate.merklePath.length === 0 + ) { invalidSyncProof('Merkle path is required') } diff --git a/packages/wallet/wallet-toolbox/src/storage/portable/index.ts b/packages/wallet/wallet-toolbox/src/storage/portable/index.ts index dd9404571..d8a9710dc 100644 --- a/packages/wallet/wallet-toolbox/src/storage/portable/index.ts +++ b/packages/wallet/wallet-toolbox/src/storage/portable/index.ts @@ -7,6 +7,7 @@ import { AESGCM, AESGCMDecrypt } from '@bsv/sdk/primitives/AESGCM' import { toArray, toBase64, toUTF8 } from '@bsv/sdk/primitives/utils' import { argon2id } from '../../utility/hashWasm' import { StorageProvider } from '../StorageProvider' +import { runInSeries } from '../../utility/runInSeries' import { TableCertificate, TableCertificateField, @@ -71,7 +72,13 @@ export interface BRC38ImportResult { updates: number } -export interface BRC39Options { +export interface BRC38ExportOptions { + /** Refuse providers without a coherent source read view. Old custom providers + * otherwise retain the caller-quiesced legacy path; built-ins use snapshots. */ + requireSnapshot?: boolean +} + +export interface BRC39Options extends BRC38ExportOptions { iterations?: number memoryKiB?: number parallelism?: number @@ -168,38 +175,51 @@ const dateFieldsByKind: Partial> = { syncState: ['created_at', 'updated_at', 'when'] } -export async function exportBRC38(storage: StorageProvider, identityKey: string): Promise { - const sourceStorage = await storage.makeAvailable() - const user = verifyTruthy(await storage.findUserByIdentityKey(identityKey)) +export async function exportBRC38( + storage: StorageProvider, + identityKey: string, + options: BRC38ExportOptions = {} +): Promise { + await storage.makeAvailable() + if (options.requireSnapshot === true || storage.supportsReadSnapshot()) { + return await storage.readSnapshot(trx => captureBRC38(storage, identityKey, trx)) + } + return await captureBRC38(storage, identityKey) +} + +async function captureBRC38( + storage: StorageProvider, + identityKey: string, + trx?: sdk.TrxToken +): Promise { + const sourceStorage = await storage.readSettings(trx) + const user = verifyTruthy(await storage.findUserByIdentityKey(identityKey, trx)) const userId = user.userId - const transactions = await storage.findTransactions({ partial: { userId } }) - const transactionIds = new Set(transactions.map(t => t.transactionId)) + const transactions = await storage.findTransactions({ partial: { userId }, trx }) const transactionTxids = new Set(transactions.map(t => t.txid).filter((txid): txid is string => txid != null)) - const provenTxReqs = (await storage.getProvenTxReqsForUser({ userId })).filter(r => transactionTxids.has(r.txid)) + const provenTxReqs = (await storage.getProvenTxReqsForUser({ userId, trx })).filter(r => transactionTxids.has(r.txid)) const provenTxIds = new Set() for (const tx of transactions) if (tx.provenTxId != null) provenTxIds.add(tx.provenTxId) for (const req of provenTxReqs) if (req.provenTxId != null) provenTxIds.add(req.provenTxId) const provenTxs: TableProvenTx[] = [] - for (const provenTxId of Array.from(provenTxIds).sort(compareNumber)) { - const proven = verifyOneOrNone(await storage.findProvenTxs({ partial: { provenTxId } })) + await runInSeries(Array.from(provenTxIds).sort(compareNumber), async provenTxId => { + const proven = verifyOneOrNone(await storage.findProvenTxs({ partial: { provenTxId }, trx })) if (proven != null) provenTxs.push(proven) - } + }) - const outputBaskets = await storage.findOutputBaskets({ partial: { userId } }) - const commissions = await storage.findCommissions({ partial: { userId } }) - const outputs = await storage.findOutputs({ partial: { userId } }) - const outputTags = await storage.findOutputTags({ partial: { userId } }) - const outputTagMaps = (await storage.getOutputTagMapsForUser({ userId })).filter(m => - outputs.some(o => o.outputId === m.outputId) - ) - const txLabels = await storage.findTxLabels({ partial: { userId } }) - const txLabelMaps = (await storage.getTxLabelMapsForUser({ userId })).filter(m => transactionIds.has(m.transactionId)) - const certificates = await storage.findCertificates({ partial: { userId } }) - const certificateFields = await storage.findCertificateFields({ partial: { userId } }) - const syncStates = await storage.findSyncStates({ partial: { userId } }) + const outputBaskets = await storage.findOutputBaskets({ partial: { userId }, trx }) + const commissions = await storage.findCommissions({ partial: { userId }, trx }) + const outputs = await storage.findOutputs({ partial: { userId }, trx }) + const outputTags = await storage.findOutputTags({ partial: { userId }, trx }) + const outputTagMaps = await storage.getOutputTagMapsForUser({ userId, trx }) + const txLabels = await storage.findTxLabels({ partial: { userId }, trx }) + const txLabelMaps = await storage.getTxLabelMapsForUser({ userId, trx }) + const certificates = await storage.findCertificates({ partial: { userId }, trx }) + const certificateFields = await storage.findCertificateFields({ partial: { userId }, trx }) + const syncStates = await storage.findSyncStates({ partial: { userId }, trx }) const data: BRC38WalletData = { brc: 38, @@ -234,8 +254,12 @@ export async function exportBRC38(storage: StorageProvider, identityKey: string) return data } -export async function exportBRC38Json(storage: StorageProvider, identityKey: string): Promise { - return canonicalize(await exportBRC38(storage, identityKey)) +export async function exportBRC38Json( + storage: StorageProvider, + identityKey: string, + options?: BRC38ExportOptions +): Promise { + return canonicalize(await exportBRC38(storage, identityKey, options)) } export function parseBRC38Json(json: string): BRC38WalletData { @@ -272,7 +296,7 @@ export async function exportBRC39( password: string, options?: BRC39Options ): Promise { - return await encryptBRC39(await exportBRC38(storage, identityKey), password, options) + return await encryptBRC39(await exportBRC38(storage, identityKey, options), password, options) } export async function importBRC39( @@ -876,7 +900,7 @@ function portableRow(kind: string, row: object): PortableRow { if (binaryFields.has(key)) { out[key] = toBase64(value as number[]) } else if (jsonFields.has(key)) { - out[key] = typeof value === 'string' ? (JSON.parse(value) as JsonValue) : (value as JsonValue) + out[key] = portableJson(typeof value === 'string' ? JSON.parse(value) : value, key) } else if (value instanceof Date) { out[key] = isoDate(value) } else { @@ -886,6 +910,38 @@ function portableRow(kind: string, row: object): PortableRow { return out } +/** Legacy JSON objects may store absent optional fields as null. Keep array + * positions and meaningful falsy values; never mutate the captured source. */ +function portableJson(value: unknown, field: string, path: Array = []): JsonValue { + if (Array.isArray(value)) return value.map((child, index) => portableJson(child, field, [...path, index])) + if (isObject(value)) { + const entries: Array<[string, JsonValue]> = [] + for (const [key, child] of Object.entries(value)) { + const childPath = [...path, key] + if (child == null && optionalPortableJsonField(field, childPath)) continue + entries.push([key, portableJson(child, field, childPath)]) + } + return Object.fromEntries(entries) + } + if (typeof value === 'string' || typeof value === 'boolean') return value + if (typeof value === 'number' && Number.isFinite(value)) return value + throw new Error('BRC-38 JSON arrays and values must not contain null, undefined or non-JSON values') +} + +function optionalPortableJsonField(field: string, path: Array): boolean { + if (field === 'history') { + return ( + (path.length === 1 && path[0] === 'notes') || + (path.length === 3 && path[0] === 'notes' && typeof path[1] === 'number' && path[2] !== 'what') + ) + } + if (field === 'notify') return path.length === 1 && path[0] === 'transactionIds' + if (field === 'syncMap') { + return path.length === 2 && SYNC_CHUNK_ENTITY_ORDER.includes(String(path[0])) && path[1] === 'maxUpdated_at' + } + return (field === 'errorLocal' || field === 'errorOther') && path.length === 1 && path[0] === 'stack' +} + function fromPortableRow(kind: string, row: PortableRow): T { const out: Record = {} const dateFields = new Set(dateFieldsByKind[kind] ?? []) diff --git a/packages/wallet/wallet-toolbox/src/storage/portable/mysqlReadSnapshot.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/portable/mysqlReadSnapshot.property.test.ts new file mode 100644 index 000000000..3fb2f1d8a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/portable/mysqlReadSnapshot.property.test.ts @@ -0,0 +1,100 @@ +import fc from 'fast-check' +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { runInSeries } from '../../utility/runInSeries' +import { mysqlReadSnapshotFixture } from '../../../test/utils/mysqlReadSnapshotFixture' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +const reads: ReadonlyArray<(storage: StorageKnex, trx: TrxToken) => Promise> = [ + (storage, trx) => storage.findProvenTxs({ partial: { provenTxId: 1 }, trx }), + (storage, trx) => storage.findProvenTxs({ partial: { txid: '11'.repeat(32) }, trx }), + (storage, trx) => storage.findSyncStates({ partial: { userId: 1, syncStateId: 1 }, trx }), + (storage, trx) => storage.findSyncStates({ partial: { userId: 1, storageIdentityKey: 'source' }, trx }), + (storage, trx) => storage.findProvenTxs({ partial: {}, trx }), + (storage, trx) => storage.findSyncStates({ partial: { userId: 1 }, trx }), + (storage, trx) => storage.findSyncStates({ partial: { syncStateId: 1 }, trx }) +] + +test('generated direct and retained view schedules preserve consistent reads, writable locks and exact failures', async () => { + await fc.assert( + fc.asyncProperty( + fc.record({ + retained: fc.boolean(), + fail: fc.boolean(), + schedule: fc.array( + fc.record({ + selector: fc.integer({ min: 0, max: reads.length - 1 }), + peer: fc.boolean(), + write: fc.boolean() + }), + { minLength: 1, maxLength: 24 } + ) + }), + async input => { + // The installed Knex transaction/query builders run against a bounded + // synthetic driver. The separate native fixture proves server isolation. + const fixture = mysqlReadSnapshotFixture() + const peer = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: fixture.db }) + peer._settings = fixture.storage.getSettings() + jest.spyOn(fixture.storage, 'readSettings').mockResolvedValue(fixture.storage.getSettings()) + const failure = new Error('generated callback failure') + const expectedLocks: boolean[] = [] + const read = async (trx: TrxToken): Promise => { + await runInSeries(input.schedule, async operation => { + const storage = operation.peer ? peer : fixture.storage + await reads[operation.selector](storage, trx) + expectedLocks.push(false) + if (operation.write) { + await fixture.db.transaction(async writable => { + await reads[operation.selector](storage, writable) + }) + // Only the four exact writable authority selectors require locks. + expectedLocks.push(operation.selector < 4) + } + }) + if (input.fail) throw failure + return input.schedule.length + } + try { + if (input.retained) { + const view = await fixture.storage.openReadSnapshot({ lifetimeMs: 30000 }) + const result = view.read(read) + if (input.fail) { + await expect(result).rejects.toBe(failure) + await expect(view.close()).rejects.toBe(failure) + } else { + await expect(result).resolves.toBe(input.schedule.length) + await view.close() + } + } else { + const result = fixture.storage.readSnapshot(read) + if (input.fail) await expect(result).rejects.toBe(failure) + else await expect(result).resolves.toBe(input.schedule.length) + } + await fixture.db.transaction(async trx => { + await fixture.storage.findProvenTxs({ partial: { provenTxId: 1 }, trx }) + }) + expectedLocks.push(true) + const actual = fixture.events.filter(sql => sql.startsWith('SELECT')) + expect(actual).toHaveLength(expectedLocks.length) + expect(actual.map(sql => sql.endsWith('FOR UPDATE'))).toEqual(expectedLocks) + expect(fixture.events.at(-1)).toBe('RELEASE') + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } + } + ), + { interruptAfterTimeLimit: 150000, markInterruptAsFailure: true } + ) +}, 180000) diff --git a/packages/wallet/wallet-toolbox/src/storage/portable/mysqlSnapshot.test.ts b/packages/wallet/wallet-toolbox/src/storage/portable/mysqlSnapshot.test.ts new file mode 100644 index 000000000..ae38fb655 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/portable/mysqlSnapshot.test.ts @@ -0,0 +1,254 @@ +import { knex, type Knex } from 'knex' +import { mysqlReadSnapshotFixture as mysqlFixture } from '../../../test/utils/mysqlReadSnapshotFixture' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' + +test('MySQL snapshot uses one reserved connection and valid next-transaction characteristics', async () => { + const fixture = mysqlFixture() + try { + const result = await fixture.storage.readSnapshot(async trx => { + await (trx as Knex.Transaction).raw('SELECT 1') + return 17 + }) + expect(result).toBe(17) + expect(fixture.acquire).toHaveBeenCalledTimes(1) + expect(fixture.events).toEqual([ + 'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY', + 'BEGIN', + 'SELECT 1', + 'COMMIT', + 'RELEASE' + ]) + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } +}) + +const exactReads = [ + ['proof id', (storage: StorageKnex, trx?: TrxToken) => storage.findProvenTxs({ partial: { provenTxId: 1 }, trx })], + [ + 'proof txid', + (storage: StorageKnex, trx?: TrxToken) => storage.findProvenTxs({ partial: { txid: '11'.repeat(32) }, trx }) + ], + [ + 'checkpoint id', + (storage: StorageKnex, trx?: TrxToken) => storage.findSyncStates({ partial: { userId: 1, syncStateId: 1 }, trx }) + ], + [ + 'checkpoint identity', + (storage: StorageKnex, trx?: TrxToken) => + storage.findSyncStates({ partial: { userId: 1, storageIdentityKey: 'source' }, trx }) + ] +] as const + +test('ordinary Postgres checkpoint metadata retains its lock while proof reads preserve their existing policy', async () => { + const fixture = mysqlFixture() + fixture.storage.getSettings().dbtype = 'Postgres' + try { + await fixture.db.transaction(async trx => { + await fixture.storage.findProvenTxs({ partial: { provenTxId: 1 }, trx }) + await fixture.storage.findSyncStates({ partial: { userId: 1, syncStateId: 1 }, trx }) + }) + const queries = fixture.events.filter(sql => sql.startsWith('SELECT')) + expect(queries).toHaveLength(2) + expect(queries.map(sql => sql.endsWith('FOR UPDATE'))).toEqual([false, true]) + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } +}) + +test('unsupported Postgres snapshots reject before opening a database transaction', async () => { + const db = knex({ client: 'pg' }) + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: db }) + const fixture = mysqlFixture() + storage._settings = { ...fixture.storage.getSettings(), dbtype: 'Postgres' } + const transaction = jest.spyOn(db.client, 'transaction') + const read = jest.fn(async () => 7) + try { + await expect(storage.readSnapshot(read)).rejects.toThrow( + 'Coherent wallet source snapshots require SQLite or MySQL isolation' + ) + expect(transaction).not.toHaveBeenCalled() + expect(read).not.toHaveBeenCalled() + } finally { + await db.destroy() + await fixture.db.destroy() + jest.restoreAllMocks() + } +}) + +test('a completed transaction is refused before the snapshot callback and its reserved connection is discarded', async () => { + const fixture = mysqlFixture() + let completed: Knex.Transaction | undefined + const original = Object.getOwnPropertyDescriptor(fixture.db, 'transaction') + if (original === undefined) throw new Error('Expected the installed Knex transaction delegate') + try { + await fixture.db.transaction(async trx => { + completed = trx + }) + if (completed === undefined) throw new Error('Expected a completed fixture transaction') + expect(completed.isCompleted()).toBe(true) + const expired = completed + fixture.events.length = 0 + // The installed Knex delegate is configurable but not writable. Preserve + // its complete descriptor while injecting a late completed callback token. + Object.defineProperty(fixture.db, 'transaction', { + ...original, + value: async (callback: (trx: Knex.Transaction) => Promise) => await callback(expired) + }) + const read = jest.fn(async () => 7) + await expect(fixture.storage.readSnapshot(read)).rejects.toThrow('MySQL snapshot transaction did not begin') + expect(read).not.toHaveBeenCalled() + expect(fixture.events).toEqual(['SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY', 'CLOSE', 'RELEASE']) + } finally { + Object.defineProperty(fixture.db, 'transaction', original) + await fixture.db.destroy() + jest.restoreAllMocks() + } +}) + +test.each(exactReads)('MySQL %s outside a transaction preserves its ordinary unlocked read', async (_name, read) => { + const fixture = mysqlFixture() + try { + await read(fixture.storage) + const queries = fixture.events.filter(sql => sql.startsWith('SELECT')) + expect(queries).toHaveLength(1) + expect(queries[0].endsWith('FOR UPDATE')).toBe(false) + expect(fixture.events).not.toContain('BEGIN') + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } +}) + +test.each([0, 1])( + 'checkpoint rows retain date and boolean normalization in both transaction modes (init=%s)', + async init => { + const when = '2026-10-03T00:00:00.000Z' + const fixture = mysqlFixture(undefined, [ + { syncStateId: 1, userId: 1, created_at: when, updated_at: when, when, init } + ]) + try { + const assertRows = (rows: Awaited>) => { + expect(rows).toHaveLength(1) + expect(rows[0]).toEqual({ + syncStateId: 1, + userId: 1, + created_at: new Date(when), + updated_at: new Date(when), + when: new Date(when), + init: init === 1 + }) + } + assertRows(await fixture.storage.findSyncStates({ partial: { userId: 1, syncStateId: 1 } })) + await fixture.storage.readSnapshot(async trx => { + assertRows(await fixture.storage.findSyncStates({ partial: { userId: 1, syncStateId: 1 }, trx })) + }) + await fixture.db.transaction(async trx => { + assertRows(await fixture.storage.findSyncStates({ partial: { userId: 1, syncStateId: 1 }, trx })) + }) + const queries = fixture.events.filter(sql => sql.startsWith('SELECT')) + expect(queries).toHaveLength(3) + expect(queries.map(sql => sql.endsWith('FOR UPDATE'))).toEqual([false, false, true]) + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } + } +) + +test.each(exactReads)( + 'MySQL snapshot %s uses a consistent read while writable transactions still lock', + async (_name, read) => { + const fixture = mysqlFixture() + try { + await fixture.storage.readSnapshot(async trx => { + await read(fixture.storage, trx) + }) + expect(fixture.events.filter(sql => sql.startsWith('SELECT'))).toHaveLength(1) + expect(fixture.events.some(sql => sql.endsWith('FOR UPDATE'))).toBe(false) + fixture.events.length = 0 + await fixture.db.transaction(async trx => { + await read(fixture.storage, trx) + }) + expect(fixture.events.filter(sql => sql.endsWith('FOR UPDATE'))).toHaveLength(1) + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } + } +) + +test('snapshot classification follows its token across providers without classifying another transaction', async () => { + const fixture = mysqlFixture() + const other = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: fixture.db }) + other._settings = fixture.storage._settings + try { + await fixture.storage.readSnapshot(async trx => { + await other.findProvenTxs({ partial: { provenTxId: 1 }, trx }) + await fixture.db.transaction(write => other.findProvenTxs({ partial: { provenTxId: 1 }, trx: write })) + await other.findProvenTxs({ partial: { provenTxId: 1 }, trx }) + }) + const selects = fixture.events.filter(sql => sql.startsWith('SELECT')) + expect(selects).toHaveLength(3) + expect(selects.map(sql => sql.endsWith('FOR UPDATE'))).toEqual([false, true, false]) + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } +}) + +test.each([false, true])( + 'retained MySQL views keep exact reads consistent through callback settlement (failure=%s)', + async fail => { + const fixture = mysqlFixture() + const failure = new Error('retained read failed') + jest.spyOn(fixture.storage, 'readSettings').mockResolvedValue(fixture.storage.getSettings()) + try { + const view = await fixture.storage.openReadSnapshot({ lifetimeMs: 30000 }) + const result = view.read(async trx => { + await fixture.storage.findProvenTxs({ partial: { provenTxId: 1 }, trx }) + await fixture.storage.findSyncStates({ partial: { userId: 1, syncStateId: 1 }, trx }) + if (fail) throw failure + return 19 + }) + if (fail) { + await expect(result).rejects.toBe(failure) + await expect(view.close()).rejects.toBe(failure) + } else { + await expect(result).resolves.toBe(19) + await view.close() + } + expect(fixture.events.filter(sql => sql.startsWith('SELECT'))).toHaveLength(2) + expect(fixture.events.some(sql => sql.endsWith('FOR UPDATE'))).toBe(false) + fixture.events.length = 0 + await fixture.db.transaction(trx => fixture.storage.findProvenTxs({ partial: { provenTxId: 1 }, trx })) + expect(fixture.events.filter(sql => sql.endsWith('FOR UPDATE'))).toHaveLength(1) + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } + } +) + +test.each(['SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY', 'BEGIN', 'SELECT 1', 'COMMIT'])( + 'MySQL snapshot discards a failed connection before returning it to the pool (%s)', + async failAt => { + const fixture = mysqlFixture(failAt) + try { + const capture = jest.fn((trx: object) => (trx as Knex.Transaction).raw('SELECT 1').then(() => undefined)) + await expect(fixture.storage.readSnapshot(capture)).rejects.toBe(fixture.failure) + await new Promise(resolve => setImmediate(resolve)) + expect(fixture.acquire).toHaveBeenCalledTimes(1) + expect(fixture.events.slice(-2)).toEqual(['CLOSE', 'RELEASE']) + if (failAt === 'SELECT 1') expect(fixture.events).toContain('ROLLBACK') + if (failAt === 'BEGIN' || failAt.startsWith('SET TRANSACTION')) expect(capture).not.toHaveBeenCalled() + } finally { + await fixture.db.destroy() + jest.restoreAllMocks() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/portable/snapshot.test.ts b/packages/wallet/wallet-toolbox/src/storage/portable/snapshot.test.ts new file mode 100644 index 000000000..1153900c9 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/portable/snapshot.test.ts @@ -0,0 +1,166 @@ +import 'fake-indexeddb/auto' +import { randomUUID } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageIdb } from '../StorageIdb' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { exportBRC38, exportBRC38Json, exportBRC39 } from './index' +import { runInSeries } from '../../utility/runInSeries' + +const identity = '02' + '11'.repeat(32) +const foreignIdentity = '03' + '22'.repeat(32) +const stores: StorageProvider[] = [] +const directories: string[] = [] + +async function sqlitePair(): Promise<[StorageKnex, StorageKnex]> { + const directory = await mkdtemp(join(tmpdir(), 'wallet-snapshot-')) + directories.push(directory) + const open = () => { + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 1000 + }) + }) + stores.push(storage) + return storage + } + const source = open() + await source.knex.raw('PRAGMA journal_mode = WAL') + await source.migrate('original source', 'source-storage') + await source.makeAvailable() + const peer = open() + await peer.makeAvailable() + return [source, peer] +} + +async function idbPair(): Promise<[StorageIdb, StorageIdb]> { + const name = `wallet-snapshot-${randomUUID()}` + const open = async () => { + const storage = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + storage.dbName = name + stores.push(storage) + await storage.migrate('original source', 'source-storage') + await storage.makeAvailable() + return storage + } + return [await open(), await open()] +} + +afterEach(async () => { + jest.restoreAllMocks() + const closed = stores.splice(0) + await runInSeries(closed, store => store.destroy()) + await runInSeries(closed.filter(store => store instanceof StorageIdb), store => store.dropAllData()) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('SQLite export holds one source view while a separate WAL writer commits between table reads', async () => { + const [source, peer] = await sqlitePair() + const { user } = await source.findOrInsertUser(identity) + await source.findOrInsertTxLabel(user.userId, 'before capture') + const find = source.findTransactions.bind(source) + jest.spyOn(source, 'findTransactions').mockImplementationOnce(async args => { + const rows = await find(args) + await peer.transaction(async trx => { + await peer.updateUser(user.userId, { activeStorage: 'later-primary', updated_at: new Date() }, trx) + await peer.insertTxLabel( + { + txLabelId: 0, + userId: user.userId, + label: 'after capture', + isDeleted: false, + created_at: new Date(), + updated_at: new Date() + }, + trx + ) + }) + return rows + }) + const captured = await exportBRC38(source, identity) + expect(captured.user.activeStorage).toBe(user.activeStorage) + expect(captured.tables.txLabels.map(row => row.label)).toEqual(['before capture']) + expect((await exportBRC38(source, identity)).tables.txLabels.map(row => row.label)).toEqual([ + 'before capture', + 'after capture' + ]) + expect((await peer.findUserByIdentityKey(identity))?.activeStorage).toBe('later-primary') +}) + +test('IndexedDB capture excludes an independently queued write and releases it after capture', async () => { + const [source, peer] = await idbPair() + const { user } = await source.findOrInsertUser(identity) + await source.findOrInsertTxLabel(user.userId, 'before capture') + const find = source.findTransactions.bind(source) + let pending: Promise | undefined + jest.spyOn(source, 'findTransactions').mockImplementationOnce(async args => { + const rows = await find(args) + pending = peer.findOrInsertTxLabel(user.userId, 'after capture') + return rows + }) + const captured = await exportBRC38(source, identity) + expect(captured.tables.txLabels.map(row => row.label)).toEqual(['before capture']) + expect(pending).toBeDefined() + await pending + expect((await exportBRC38(source, identity)).tables.txLabels.map(row => row.label)).toEqual([ + 'before capture', + 'after capture' + ]) +}) + +test.each([sqlitePair, idbPair])( + 'source metadata comes from the database view and wallet profiles remain separate (%#)', + async pair => { + const [source] = await pair() + const { user } = await source.findOrInsertUser(identity) + const { user: foreign } = await source.findOrInsertUser(foreignIdentity) + await source.findOrInsertTxLabel(user.userId, 'own label') + await source.findOrInsertTxLabel(foreign.userId, 'foreign label') + source.getSettings().storageName = 'stale cached name' + const captured = await exportBRC38(source, identity) + expect(captured.sourceStorage.storageName).toBe('original source') + expect(captured.user.identityKey).toBe(identity) + expect(captured.tables.txLabels.map(row => row.label)).toEqual(['own label']) + expect(source.getSettings().storageName).toBe('stale cached name') + } +) + +test.each([sqlitePair, idbPair])('capture failure releases the view without writing wallet data (%#)', async pair => { + const [source] = await pair() + const { user } = await source.findOrInsertUser(identity) + const failure = new Error('capture stopped') + const read = jest.spyOn(source, 'findOutputs').mockRejectedValueOnce(failure) + await expect(exportBRC38(source, identity)).rejects.toBe(failure) + read.mockRestore() + await source.findOrInsertTxLabel(user.userId, 'write after failure') + expect((await exportBRC38(source, identity)).tables.txLabels.map(row => row.label)).toEqual(['write after failure']) +}) + +test('a provider without a coherent read implementation refuses a complete source export', async () => { + const [source] = await sqlitePair() + await source.findOrInsertUser(identity) + jest.spyOn(source, 'readSnapshot').mockImplementation(StorageProvider.prototype.readSnapshot) + jest.spyOn(source, 'supportsReadSnapshot').mockReturnValue(false) + const reads = jest.spyOn(source, 'findTransactions') + await expect(exportBRC38(source, identity, { requireSnapshot: true })).rejects.toThrow( + 'Coherent wallet source snapshots are not supported' + ) + await expect(exportBRC38Json(source, identity, { requireSnapshot: true })).rejects.toThrow( + 'Coherent wallet source snapshots are not supported' + ) + await expect(exportBRC39(source, identity, 'synthetic password', { requireSnapshot: true })).rejects.toThrow( + 'Coherent wallet source snapshots are not supported' + ) + expect(reads).not.toHaveBeenCalled() + const legacy = await exportBRC38(source, identity) + expect(legacy.user.identityKey).toBe(identity) + expect(reads).toHaveBeenCalledTimes(1) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClient.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClient.ts index 7522559de..408c201e5 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClient.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClient.ts @@ -103,8 +103,8 @@ export class StorageClient extends StorageClientBase { * @param method The WalletStorage method name to call. * @param params The array of parameters to pass to the method in order. */ - protected async rpcCall(method: string, params: unknown[]): Promise { - return await this.traceRpcCall(method, params, async rpcSpan => { + protected rpcCall(method: string, params: unknown[]): Promise { + return this.traceRpcCall(method, params, async rpcSpan => { const loggerState = this.startRpcLogging(method, params) const { logger } = loggerState @@ -131,8 +131,8 @@ export class StorageClient extends StorageClientBase { response = await this.traceRpcStep( 'wallet.storage.http', rpcSpan, - async () => - await this.authenticatedFetch(this.endpointUrl, { + () => + this.authenticatedFetch(this.endpointUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -157,15 +157,10 @@ export class StorageClient extends StorageClientBase { const responseUsesBinary = response.headers.get(BINARY_ENCODING_HEADER) === BINARY_ENCODING if (responseUsesBinary) this.serverSupportsBinary = true - const responseText = await this.traceRpcStep( - 'wallet.storage.response.read', - rpcSpan, - async () => await response.text(), - { - 'http.response.status_code': response.status, - 'rpc.encoding': responseUsesBinary ? 'binary-json' : 'json' - } - ) + const responseText = await this.traceRpcStep('wallet.storage.response.read', rpcSpan, () => response.text(), { + 'http.response.status_code': response.status, + 'rpc.encoding': responseUsesBinary ? 'binary-json' : 'json' + }) const json = await this.traceRpcStep( 'wallet.storage.response.parse', rpcSpan, diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts index bb326b5b1..6ef8bc192 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageClientBase.ts @@ -73,7 +73,7 @@ import { StorageCapabilities } from '../../sdk/ActionBatch.interfaces' import { TableSettings } from '../schema/tables/TableSettings' -import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { WERR_INVALID_OPERATION, WERR_NOT_IMPLEMENTED } from '../../sdk/WERR_errors' import { WalletServices } from '../../sdk/WalletServices.interfaces' import { TableUser } from '../schema/tables/TableUser' import { TableSyncState } from '../schema/tables/TableSyncState' @@ -91,6 +91,20 @@ import { supportedActionBatchPackEncodings } from '../../utility/actionBatchPack' import { pruneBeefForTxids } from '../../utility/beefForTxids' +import { SnapshotArchiveTransport } from '../snapshot/archive/SnapshotArchiveTransport' +import { openRemoteSnapshot } from '../snapshot/archive/openRemoteSnapshot' +import { SnapshotCancelledError } from '../snapshot/SnapshotCancelledError' +import { snapshotArchiveFetch } from '../snapshot/archive/SnapshotArchiveTransportFailure' +import type { SnapshotSyncStorage } from '../snapshot/SnapshotSync' +import { + snapshotArchiveResponseBytes, + snapshotArchiveRequestBytes, + validateSnapshotArchiveCapabilities, + type SnapshotArchiveCapabilities, + type SnapshotArchiveMethod +} from '../snapshot/archive/SnapshotArchiveProtocol' +import { BINARY_ENCODING, BINARY_ENCODING_HEADER, parseJsonRpc, validateJsonRpcResponse } from './BinaryJson' +import { WalletErrorFromJson } from '../../sdk/WalletErrorFromJson' const syncChunkResponseRetryLimit = 4 const minimumSyncChunkRoughSize = 64 * 1024 @@ -103,9 +117,13 @@ type RemoteStorageSettings = TableSettings & { /** Runtime-only RPC advertisement, not a persisted settings-table column. */ syncCheckpointVersion?: 1 syncTransfer?: SyncTransferCapabilities + snapshotArchive?: SnapshotArchiveCapabilities + snapshotArchiveReaderVersion?: 1 } export interface StorageClientOptions { + /** Disable immutable remote snapshots during a mixed-version rollout. */ + snapshotArchives?: boolean /** * Send compact tagged binary request values after the server advertises * support. Leave disabled during rolling deployments where an endpoint may @@ -253,6 +271,13 @@ function validateRemoteStorageSettings(value: unknown): RemoteStorageSettings { if (properties.syncCheckpointVersion != null && properties.syncCheckpointVersion.value !== 1) { throw new Error('Wallet storage returned invalid settings.') } + if (properties.snapshotArchive != null) validateSnapshotArchiveCapabilities(properties.snapshotArchive.value) + if ( + properties.snapshotArchiveReaderVersion != null && + (properties.snapshotArchiveReaderVersion.value !== 1 || properties.snapshotArchive == null) + ) { + throw new Error('Wallet storage returned invalid settings.') + } return value as RemoteStorageSettings } @@ -273,6 +298,10 @@ export abstract class StorageClientBase implements WalletStorageProvider { private authenticatedServerIdentityKey?: string private readonly expectedStorageIdentityKey?: string private syncChunkRoughSizeLimit?: number + private readonly snapshotArchivesEnabled: boolean + private readonly snapshotWallet: WalletInterface + private snapshotAuthClient?: AuthFetch + private snapshotSource?: { identityKey: string; chain: 'main' | 'test'; supportsReader: boolean } /** Optional progress/cancellation hook for a bounded transfer; never receives wallet contents. */ onSyncTransferProgress?: (progress: { direction: 'read' | 'write'; bytes: number; totalBytes: number }) => void @@ -281,6 +310,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { constructor(wallet: WalletInterface, endpointUrl: string, options: StorageClientOptions = {}) { this.authClient = new AuthFetch(wallet) + this.snapshotWallet = wallet + this.snapshotArchivesEnabled = options.snapshotArchives !== false this.endpointUrl = normalizeStorageEndpointUrl(endpointUrl) this.binaryRequests = options.binaryRequests === true this.telemetry = new Telemetry(options.telemetry) @@ -291,7 +322,10 @@ export abstract class StorageClientBase implements WalletStorageProvider { } protected async authenticatedFetch(url: string, config: Parameters[1]): Promise { - const response = await this.authClient.fetch(url, config) + return this.validateAuthenticatedResponse(await this.authClient.fetch(url, config)) + } + + private validateAuthenticatedResponse(response: Response): Response { const authenticatedIdentityKey = response.headers.get('x-bsv-auth-identity-key') if (authenticatedIdentityKey == null) { throw new Error('Wallet storage response was not mutually authenticated.') @@ -304,6 +338,82 @@ export abstract class StorageClientBase implements WalletStorageProvider { return response } + /** Source-only adapter. Opening performs authenticated capability negotiation, including on the first sync. */ + getSnapshotSync(): SnapshotSyncStorage | undefined { + if (!this.snapshotArchivesEnabled) return undefined + const unavailable = () => Promise.reject(new WERR_NOT_IMPLEMENTED('Remote snapshot destination is unavailable')) + return { + fallbackOnResourceError: false, + supportsDestination: () => Promise.resolve(false), + openSource: async (identityKey, options) => { + const transport = await this.getSnapshotArchiveTransport(identityKey) + return transport === undefined ? undefined : await openRemoteSnapshot(transport, options) + }, + begin: unavailable, + checkpoint: unavailable, + prepare: unavailable + } + } + + /** Available only after an authenticated compatible advertisement. */ + async getSnapshotArchiveTransport(identityKey: string): Promise { + await this.makeAvailable() + if (!this.snapshotArchivesEnabled || this.snapshotSource === undefined) return undefined + return new SnapshotArchiveTransport( + (method, params, signal) => this.snapshotRpcCall(method, params, signal), + identityKey, + this.snapshotSource.identityKey, + this.snapshotSource.chain, + this.snapshotSource.supportsReader + ) + } + + private async snapshotRpcCall( + method: SnapshotArchiveMethod, + params: unknown[], + signal?: AbortSignal + ): Promise { + const id = this.nextRequestId() + const body = JSON.stringify({ jsonrpc: '2.0', method, params, id }) + if (new TextEncoder().encode(body).length > snapshotArchiveRequestBytes) + throw new TypeError('Snapshot archive request exceeds its transport limit') + this.snapshotAuthClient ??= new AuthFetch( + this.snapshotWallet, + undefined, + undefined, + undefined, + { maxResponseBytes: snapshotArchiveResponseBytes }, + snapshotArchiveFetch(fetch) + ) + let received: Response + try { + received = await this.snapshotAuthClient.fetch(this.endpointUrl, { + method: 'POST', + headers: { 'Content-Type': 'application/json', [BINARY_ENCODING_HEADER]: BINARY_ENCODING }, + body, + signal + }) + } catch (error) { + // AuthFetch uses its typed cancellation code before a native fetch may + // exist. Inspect only its own data code here, before RPC/response parsing; + // this also works with older SDKs that lack the exported error class. + if ( + signal?.aborted === true && + error instanceof Error && + Object.getOwnPropertyDescriptor(error, 'code')?.value === 'ERR_PAYMENT_CANCELLED' + ) + throw new SnapshotCancelledError('Snapshot archive authenticated request was cancelled') + throw error + } + const response = this.validateAuthenticatedResponse(received) + if (!response.ok) throw this.rpcResponseError(response) + if (response.headers.get(BINARY_ENCODING_HEADER) !== BINARY_ENCODING) + throw new TypeError('Snapshot archive requires compact binary responses') + const json = validateJsonRpcResponse(parseJsonRpc(await response.text(), true), id) + if ('error' in json) throw WalletErrorFromJson(json.error as object) + return json.result + } + protected async traceRpcCall( method: string, params: unknown[], @@ -364,6 +474,11 @@ export abstract class StorageClientBase implements WalletStorageProvider { */ protected abstract rpcCall(method: string, params: unknown[]): Promise + /** Share the asynchronous rejection boundary, including custom transports that throw synchronously. */ + private async forwardRpc(method: string, params: unknown[]): Promise { + return this.rpcCall(method, params) + } + protected nextRequestId(): number { if (!Number.isSafeInteger(this.nextId) || this.nextId < 1) { throw new Error('Wallet storage request identifier space exhausted.') @@ -405,6 +520,13 @@ export abstract class StorageClientBase implements WalletStorageProvider { if (this.expectedStorageIdentityKey !== undefined && storageIdentityKey !== this.expectedStorageIdentityKey) { throw new Error('Wallet storage settings identity does not match the configured storage identity.') } + if (settings.snapshotArchive !== undefined && (settings.chain === 'main' || settings.chain === 'test')) { + this.snapshotSource = { + identityKey: storageIdentityKey, + chain: settings.chain, + supportsReader: settings.snapshotArchiveReaderVersion === 1 + } + } this.settings = settings return this.settings } @@ -420,8 +542,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { /** * Called to cleanup resources when no further use of this object will occur. */ - async destroy(): Promise { - return await this.rpcCall('destroy', []) + destroy(): Promise { + return this.forwardRpc('destroy', []) } /** @@ -431,8 +553,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param storageIdentityKey Unique identity key for remote storage if it does not yet exist. * @returns current schema migration identifier */ - async migrate(storageName: string, _storageIdentityKey: string): Promise { - return await this.rpcCall('migrate', [storageName]) + migrate(storageName: string, _storageIdentityKey: string): Promise { + return this.forwardRpc('migrate', [storageName]) } /** @@ -464,8 +586,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param args Original wallet `internalizeAction` arguments. * @returns `internalizeAction` results */ - async internalizeAction(auth: AuthId, args: InternalizeActionArgs): Promise { - return await this.rpcCall('internalizeAction', [auth, args]) + internalizeAction(auth: AuthId, args: InternalizeActionArgs): Promise { + return this.forwardRpc('internalizeAction', [auth, args]) } /** @@ -518,47 +640,47 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param args `StorageProcessActionArgs` convey completed signed transaction to storage. * @returns `StorageProcessActionResults` supporting final wallet processing to yield `createAction` or `signAction` results. */ - async processAction(auth: AuthId, args: StorageProcessActionArgs): Promise { - return await this.rpcCall('processAction', [auth, args]) + processAction(auth: AuthId, args: StorageProcessActionArgs): Promise { + return this.forwardRpc('processAction', [auth, args]) } - async prepareNoSendExpiry(auth: AuthId, args: ValidCreateActionArgs): Promise { - return await this.rpcCall('prepareNoSendExpiry', [auth, args]) + prepareNoSendExpiry(auth: AuthId, args: ValidCreateActionArgs): Promise { + return this.forwardRpc('prepareNoSendExpiry', [auth, args]) } - async activateNoSendExpiry( + activateNoSendExpiry( auth: AuthId, args: StorageActivateNoSendExpiryArgs ): Promise { - return await this.rpcCall('activateNoSendExpiry', [auth, args]) + return this.forwardRpc('activateNoSendExpiry', [auth, args]) } async armNoSendExpiry(auth: AuthId, args: StorageArmNoSendExpiryArgs): Promise { await this.rpcCall('armNoSendExpiry', [auth, args]) } - async getCapabilities(): Promise { - return await this.rpcCall('getCapabilities', []) + getCapabilities(): Promise { + return this.forwardRpc('getCapabilities', []) } - async beginActionBatch(auth: AuthId, args: BeginActionBatchArgs): Promise { - return await this.rpcCall('beginActionBatch', [auth, args]) + beginActionBatch(auth: AuthId, args: BeginActionBatchArgs): Promise { + return this.forwardRpc('beginActionBatch', [auth, args]) } - async extendActionBatch(auth: AuthId, args: ExtendActionBatchArgs): Promise { - return await this.rpcCall('extendActionBatch', [auth, args]) + extendActionBatch(auth: AuthId, args: ExtendActionBatchArgs): Promise { + return this.forwardRpc('extendActionBatch', [auth, args]) } - async renewActionBatch(auth: AuthId, batchId: string): Promise { - return await this.rpcCall('renewActionBatch', [auth, batchId]) + renewActionBatch(auth: AuthId, batchId: string): Promise { + return this.forwardRpc('renewActionBatch', [auth, batchId]) } - async resumeActionBatch(auth: AuthId, args: ResumeActionBatchArgs): Promise { - return await this.rpcCall('resumeActionBatch', [auth, args]) + resumeActionBatch(auth: AuthId, args: ResumeActionBatchArgs): Promise { + return this.forwardRpc('resumeActionBatch', [auth, args]) } - async prepareActionBatchCommit(auth: AuthId, manifest: ActionBatchManifest): Promise { - return await this.rpcCall('prepareActionBatchCommit', [auth, manifest]) + prepareActionBatchCommit(auth: AuthId, manifest: ActionBatchManifest): Promise { + return this.forwardRpc('prepareActionBatchCommit', [auth, manifest]) } async putActionBatchBlob(auth: AuthId, args: PutActionBatchBlobArgs): Promise { @@ -600,16 +722,16 @@ export abstract class StorageClientBase implements WalletStorageProvider { } } - async commitActionBatch(auth: AuthId, manifest: ActionBatchManifest): Promise { - return await this.rpcCall('commitActionBatch', [auth, manifest]) + commitActionBatch(auth: AuthId, manifest: ActionBatchManifest): Promise { + return this.forwardRpc('commitActionBatch', [auth, manifest]) } - async commitActionBatchByDigest(auth: AuthId, args: CommitActionBatchByDigestArgs): Promise { - return await this.rpcCall('commitActionBatchByDigest', [auth, args]) + commitActionBatchByDigest(auth: AuthId, args: CommitActionBatchByDigestArgs): Promise { + return this.forwardRpc('commitActionBatchByDigest', [auth, args]) } - async abortActionBatch(auth: AuthId, batchId: string): Promise { - return await this.rpcCall('abortActionBatch', [auth, batchId]) + abortActionBatch(auth: AuthId, batchId: string): Promise { + return this.forwardRpc('abortActionBatch', [auth, batchId]) } /** @@ -619,8 +741,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param args original wallet `abortAction` args. * @returns `abortAction` result. */ - async abortAction(auth: AuthId, args: AbortActionArgs): Promise { - return await this.rpcCall('abortAction', [auth, args]) + abortAction(auth: AuthId, args: AbortActionArgs): Promise { + return this.forwardRpc('abortAction', [auth, args]) } /** @@ -629,8 +751,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param identityKey of the user. * @returns `TableUser` for the user and whether a new user was created. */ - async findOrInsertUser(identityKey: string): Promise<{ user: TableUser; isNew: boolean }> { - return await this.rpcCall<{ user: TableUser; isNew: boolean }>('findOrInsertUser', [identityKey]) + findOrInsertUser(identityKey: string): Promise<{ user: TableUser; isNew: boolean }> { + return this.forwardRpc<{ user: TableUser; isNew: boolean }>('findOrInsertUser', [identityKey]) } /** Read compact progress only when the provider advertises support. */ @@ -675,9 +797,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param certificate the certificate to insert. * @returns record Id of the inserted `TableCertificate` record. */ - async insertCertificateAuth(auth: AuthId, certificate: TableCertificateX): Promise { - const r = await this.rpcCall('insertCertificateAuth', [auth, certificate]) - return r + insertCertificateAuth(auth: AuthId, certificate: TableCertificateX): Promise { + return this.forwardRpc('insertCertificateAuth', [auth, certificate]) } /** @@ -687,9 +808,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param args Validated extension of original wallet `listActions` arguments. * @returns `listActions` results. */ - async listActions(auth: AuthId, vargs: ValidListActionsArgs): Promise { - const r = await this.rpcCall('listActions', [auth, vargs]) - return r + listActions(auth: AuthId, vargs: ValidListActionsArgs): Promise { + return this.forwardRpc('listActions', [auth, vargs]) } /** @@ -699,9 +819,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param args Validated extension of original wallet `listOutputs` arguments. * @returns `listOutputs` results. */ - async listOutputs(auth: AuthId, vargs: ValidListOutputsArgs): Promise { - const r = await this.rpcCall('listOutputs', [auth, vargs]) - return r + listOutputs(auth: AuthId, vargs: ValidListOutputsArgs): Promise { + return this.forwardRpc('listOutputs', [auth, vargs]) } /** @@ -711,9 +830,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param args Validated extension of original wallet `listCertificates` arguments. * @returns `listCertificates` results. */ - async listCertificates(auth: AuthId, vargs: ValidListCertificatesArgs): Promise { - const r = await this.rpcCall('listCertificates', [auth, vargs]) - return r + listCertificates(auth: AuthId, vargs: ValidListCertificatesArgs): Promise { + return this.forwardRpc('listCertificates', [auth, vargs]) } /** @@ -796,8 +914,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * This must match the `AuthFetch` identity securing the remote conneciton. * @param args original wallet `relinquishCertificate` args. */ - async relinquishCertificate(auth: AuthId, args: RelinquishCertificateArgs): Promise { - return await this.rpcCall('relinquishCertificate', [auth, args]) + relinquishCertificate(auth: AuthId, args: RelinquishCertificateArgs): Promise { + return this.forwardRpc('relinquishCertificate', [auth, args]) } /** @@ -809,8 +927,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * This must match the `AuthFetch` identity securing the remote conneciton. * @param args original wallet `relinquishOutput` args. */ - async relinquishOutput(auth: AuthId, args: RelinquishOutputArgs): Promise { - return await this.rpcCall('relinquishOutput', [auth, args]) + relinquishOutput(auth: AuthId, args: RelinquishOutputArgs): Promise { + return this.forwardRpc('relinquishOutput', [auth, args]) } /** @@ -833,7 +951,9 @@ export abstract class StorageClientBase implements WalletStorageProvider { bytes != null && bytes.length * expansion > (capabilities!.inlineBytes ?? 6 * 1024 * 1024) ? await this.uploadSyncTransfer(args.identityKey, bytes, capabilities!) : await this.rpcCall('processSyncChunk', [args, wireChunk]) - if (r.nextCheckpoint != null) r.nextCheckpoint = validateSyncCheckpoint(r.nextCheckpoint, args) + // Preserve the failure result; its error takes precedence over checkpoint fields. + if (r.error == null && r.nextCheckpoint != null) + r.nextCheckpoint = validateSyncCheckpoint(r.nextCheckpoint, args, r.done === true) return r } @@ -1037,11 +1157,10 @@ export abstract class StorageClientBase implements WalletStorageProvider { * @param args proof request and new transaction proof data * @returns results of updates */ - async updateProvenTxReqWithNewProvenTx( + updateProvenTxReqWithNewProvenTx( args: UpdateProvenTxReqWithNewProvenTxArgs ): Promise { - const r = await this.rpcCall('updateProvenTxReqWithNewProvenTx', [args]) - return r + return this.forwardRpc('updateProvenTxReqWithNewProvenTx', [args]) } /** @@ -1053,8 +1172,8 @@ export abstract class StorageClientBase implements WalletStorageProvider { * This must match the `AuthFetch` identity securing the remote conneciton. * @param newActiveStorageIdentityKey which must be a currently configured backup storage provider. */ - async setActive(auth: AuthId, newActiveStorageIdentityKey: string): Promise { - return await this.rpcCall('setActive', [auth, newActiveStorageIdentityKey]) + setActive(auth: AuthId, newActiveStorageIdentityKey: string): Promise { + return this.forwardRpc('setActive', [auth, newActiveStorageIdentityKey]) } /** @see {@link validateDate} */ diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageMobile.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageMobile.ts index 0328cbf8f..ffaa1d7f1 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageMobile.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageMobile.ts @@ -41,8 +41,8 @@ export class StorageClient extends StorageClientBase { * @param method The WalletStorage method name to call. * @param params The array of parameters to pass to the method in order. */ - protected async rpcCall(method: string, params: unknown[]): Promise { - return await this.traceRpcCall(method, params, async rpcSpan => { + protected rpcCall(method: string, params: unknown[]): Promise { + return this.traceRpcCall(method, params, async rpcSpan => { const id = this.nextRequestId() const body = { jsonrpc: '2.0', @@ -61,8 +61,8 @@ export class StorageClient extends StorageClientBase { const response = await this.traceRpcStep( 'wallet.storage.http', rpcSpan, - async () => - await this.authenticatedFetch(this.endpointUrl, { + () => + this.authenticatedFetch(this.endpointUrl, { method: 'POST', headers: { 'Content-Type': 'application/json', @@ -83,15 +83,10 @@ export class StorageClient extends StorageClientBase { const responseUsesBinary = response.headers.get(BINARY_ENCODING_HEADER) === BINARY_ENCODING if (responseUsesBinary) this.serverSupportsBinary = true - const responseText = await this.traceRpcStep( - 'wallet.storage.response.read', - rpcSpan, - async () => await response.text(), - { - 'http.response.status_code': response.status, - 'rpc.encoding': responseUsesBinary ? 'binary-json' : 'json' - } - ) + const responseText = await this.traceRpcStep('wallet.storage.response.read', rpcSpan, () => response.text(), { + 'http.response.status_code': response.status, + 'rpc.encoding': responseUsesBinary ? 'binary-json' : 'json' + }) const json = await this.traceRpcStep( 'wallet.storage.response.parse', rpcSpan, diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts index 8ff0ceeb8..9aee927f3 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/StorageServer.ts @@ -30,7 +30,7 @@ import type { PaymentReplayStore } from '@bsv/payment-express-middleware' import { Options as RateLimitOptions, rateLimit } from 'express-rate-limit' import { Wallet } from '../../Wallet' import { StorageProvider } from '../StorageProvider' -import { WERR_INTERNAL, WERR_NOT_ACTIVE, WERR_UNAUTHORIZED } from '../../sdk/WERR_errors' +import { WERR_INTERNAL, WERR_INVALID_OPERATION, WERR_NOT_ACTIVE, WERR_UNAUTHORIZED } from '../../sdk/WERR_errors' import { AuthId, SyncChunk } from '../../sdk/WalletStorage.interfaces' import { EntityTimeStamp } from '../../sdk/types' import { validateDate, validateEntity, validateEntities, validateSyncChunkEntities } from './entityValidationHelpers' @@ -67,6 +67,13 @@ import { } from '../../utility/actionBatchPack' import { ACTION_BATCH_MAX_PACK_BYTES, ACTION_BATCH_MAX_PACK_ITEMS } from '../methods/actionBatchBlobs' import { validateSyncProofs } from './validateRpcSyncProofs' +import { KnexSnapshotArchiveRpc } from '../snapshot/archive/KnexSnapshotArchiveRpc' +import { + snapshotArchiveMethods, + snapshotArchiveRequestBytes, + snapshotArchiveResponseBytes, + type SnapshotArchiveMethod +} from '../snapshot/archive/SnapshotArchiveProtocol' const storageRpcMethods = new Set([ 'abortAction', @@ -270,12 +277,18 @@ export interface WalletStorageServerOptions { maxRpcResponseBytes?: number /** Disable the additive durable transfer transport during a mixed-version rollout. Knex only. */ syncTransfers?: boolean + /** Disable immutable snapshot archives during a mixed-version rollout. Requires migrated WAL/MySQL storage and snapshotSync. */ + snapshotArchives?: boolean /** Durable BRC-105 replay claims for monetized multi-replica deployments. */ paymentReplayStore?: PaymentReplayStore } export class StorageServer { private readonly syncTransfers?: KnexSyncTransferStore + private readonly snapshotArchivesEnabled: boolean + private snapshotArchives?: KnexSnapshotArchiveRpc + private closing?: Promise + private closed = false private readonly app = express() private readonly host?: string private readonly port: number @@ -396,6 +409,11 @@ export class StorageServer { highThroughput: 32 * 1024 * 1024 }) ) + this.snapshotArchivesEnabled = + options.snapshotArchives !== false && + jsonBodyLimit >= snapshotArchiveRequestBytes && + (this.maxRpcResponseBytes === -1 || this.maxRpcResponseBytes >= snapshotArchiveResponseBytes) + this.snapshotArchives = this.createSnapshotArchiveRpc() // Keep legacy configurations working when their envelopes cannot fit even a minimum part. if ( options.syncTransfers !== false && @@ -640,10 +658,14 @@ export class StorageServer { const logObj = this.createRpcLog(req, method, id, params) try { this.enforceRpcRequestBudgets(method, params) - const dispatch = - method.endsWith('SyncTransfer') || method.endsWith('SyncTransferPart') - ? await this.dispatchSyncTransfer(method, params, req) - : await this.dispatchRpcCall(method, params, req, logObj, rpcSpan) + let dispatch: RpcDispatchResult + if (snapshotArchiveMethods.includes(method as SnapshotArchiveMethod)) { + dispatch = await this.dispatchSnapshotArchive(method as SnapshotArchiveMethod, params, req, useBinary, id) + } else if (method.endsWith('SyncTransfer') || method.endsWith('SyncTransferPart')) { + dispatch = await this.dispatchSyncTransfer(method, params, req) + } else { + dispatch = await this.dispatchRpcCall(method, params, req, logObj, rpcSpan) + } if (!dispatch.found) { return this.sendRpc( res, @@ -663,6 +685,12 @@ export class StorageServer { // JSON.stringify silently omit an undefined result. const payload = { jsonrpc: '2.0', result: result ?? null, id } const serialized = escapeRpcJson(stringifyJsonRpc(payload, useBinary)) + if ( + snapshotArchiveMethods.includes(method as SnapshotArchiveMethod) && + Buffer.byteLength(serialized, 'utf8') > snapshotArchiveResponseBytes + ) { + throw new WERR_INVALID_OPERATION('Snapshot archive response exceeds its transport limit') + } // Apply the response bound before consulting any client transport preference. if (this.maxRpcResponseBytes !== -1 && Buffer.byteLength(serialized, 'utf8') > this.maxRpcResponseBytes) { return await this.sendOversizedSyncResponse(req, res, useBinary, method, params, payload) @@ -932,7 +960,8 @@ export class StorageServer { result = { ...result, syncCheckpointVersion: 1, - ...(this.syncTransfers == null ? {} : { syncTransfer: this.syncTransfers.capabilities }) + ...(this.syncTransfers == null ? {} : { syncTransfer: this.syncTransfers.capabilities }), + ...(await this.snapshotArchiveSettings()) } } this.finishRpcLogging(logger, result) @@ -944,6 +973,11 @@ export class StorageServer { } } + private async snapshotArchiveSettings(): Promise> { + const snapshotArchive = await this.snapshotArchives?.capabilities() + return snapshotArchive === undefined ? {} : { snapshotArchive } + } + private async dispatchSyncTransfer(method: string, params: any[], req: Request): Promise { const transfers = this.syncTransfers if (transfers == null) return { found: false } @@ -1011,6 +1045,36 @@ export class StorageServer { return { found: true, result } } + private createSnapshotArchiveRpc(): KnexSnapshotArchiveRpc | undefined { + return this.snapshotArchivesEnabled && this.storage instanceof StorageKnex + ? new KnexSnapshotArchiveRpc(this.storage) + : undefined + } + + private async dispatchSnapshotArchive( + method: SnapshotArchiveMethod, + params: unknown[], + req: Request, + useBinary: boolean, + id: unknown + ): Promise { + if (this.snapshotArchives === undefined) return { found: false } + // The shared JSON parser already bounds ingress. This smaller post-parse + // envelope check is an additional protocol bound, not a pre-parse claim. + if ( + !useBinary || + !Number.isSafeInteger(id) || + Number(id) < 1 || + Buffer.byteLength(JSON.stringify(req.body), 'utf8') > snapshotArchiveRequestBytes + ) { + throw new WERR_INVALID_OPERATION('Snapshot archive requests require bounded compact-binary RPC framing') + } + return { + found: true, + result: await this.snapshotArchives.dispatch(method, params, requiredAuthenticatedIdentityKey(req)) + } + } + private async traceRpcStep( name: string, parent: TelemetrySpan | undefined, @@ -1165,6 +1229,12 @@ export class StorageServer { server: any public start(): void { + if (this.closing !== undefined) { + if (!this.closed) throw new WERR_INVALID_OPERATION('Storage server is closing') + this.closing = undefined + this.closed = false + this.snapshotArchives = this.createSnapshotArchiveRpc() + } const listening = (): void => { console.log(`WalletStorageServer listening at http://${this.host ?? 'localhost'}:${this.port}`) } @@ -1173,12 +1243,25 @@ export class StorageServer { configureHttpServer(this.server, 'WALLET_STORAGE', this.httpPolicy) } - public async close(): Promise { - if (this.server) { - await this.server.close(() => { - // console.log('WalletStorageServer closed') + public close(): Promise { + if (this.closing !== undefined) return this.closing + // Fence capture admission synchronously, then await BOTH physical drains. + const captures = this.snapshotArchives?.close() ?? Promise.resolve() + const http = new Promise((resolve, reject) => { + if (!this.server) { + resolve() + return + } + this.server.close((error?: NodeJS.ErrnoException) => { + if (error !== undefined && error.code !== 'ERR_SERVER_NOT_RUNNING') reject(error) + else resolve() }) - } + }) + this.closing = Promise.allSettled([captures, http]).then(results => { + for (const result of results) if (result.status === 'rejected') throw result.reason + this.closed = true + }) + return this.closing } /** @see {@link validateDate} */ diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.security.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.security.test.ts index 361e6ec8c..11899ab58 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.security.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.security.test.ts @@ -22,6 +22,17 @@ describe.each([ expect(() => new Client(wallet, 'http://wallet.localhost:8042')).not.toThrow() }) + test.each([0x00, 0x1f, 0x7f, 0x85, 0x9f])('rejects a control code unit %i anywhere in an endpoint', code => { + expect(() => new Client(wallet, `https://storage.example.com/rpc${String.fromCharCode(code)}suffix`)).toThrow( + 'exact bounded URL' + ) + }) + + test('preserves non-ASCII and supplementary Unicode endpoint paths', () => { + const endpoint = 'https://storage.example.com/é/😀' + expect(new Client(wallet, endpoint).endpointUrl).toBe(endpoint) + }) + test('rejects ambiguous endpoint components', () => { expect(() => new Client(wallet, '/rpc')).toThrow('absolute URL') expect(() => new Client(wallet, 'https://user:pass@storage.example.com/rpc')).toThrow( diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.telemetry.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.telemetry.test.ts index f85d5148e..f72040490 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.telemetry.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClient.telemetry.test.ts @@ -77,6 +77,45 @@ describe('StorageClient telemetry', () => { } ) + describe.each([ + ['browser and Node', StorageClient], + ['mobile', StorageMobile] + ])('%s callback failures', (_name, Client) => { + it.each([false, true])('preserves response-read rejection with telemetry enabled=%s', async enabled => { + const events: TelemetryEvent[] = [] + const client = new Client( + {} as WalletInterface, + 'https://storage.example.test/rpc', + enabled ? { telemetry: { sink: { capture: (event: TelemetryEvent) => events.push(event) } } } : {} + ) + const error = new Error('response read failed') + const text = jest.fn(() => { + throw error + }) + Reflect.set(client, 'authClient', { + fetch: jest.fn(async () => ({ + ok: true, + status: 200, + headers: new Headers(AUTHENTICATED_HEADERS), + text + })) + }) + + const result = Reflect.get(client, 'rpcCall').call(client, 'isAvailable', [{ userId: 1 }]) + expect(result).toBeInstanceOf(Promise) + await expect(result).rejects.toBe(error) + expect(text).toHaveBeenCalledTimes(1) + if (enabled) { + expect(events.filter(event => event.spanStatus === 'error').map(event => event.name)).toEqual([ + 'wallet.storage.response.read', + 'wallet.storage.rpc' + ]) + } else { + expect(events).toEqual([]) + } + }) + }) + it('preserves caller logging while reporting remote and protocol failures', async () => { const events: TelemetryEvent[] = [] const client = new StorageClient({} as WalletInterface, 'https://storage.example.test/rpc', { diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.forwarding.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.forwarding.test.ts new file mode 100644 index 000000000..f6d0d3b91 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.forwarding.test.ts @@ -0,0 +1,103 @@ +import type { WalletInterface } from '@bsv/sdk' +import { StorageClientBase } from '../StorageClientBase' + +class ForwardingClient extends StorageClientBase { + constructor(private readonly send: (method: string, params: unknown[]) => Promise) { + super({} as WalletInterface, 'https://storage.example.test') + } + + protected rpcCall(method: string, params: unknown[]): Promise { + return this.send(method, params) as Promise + } +} + +const auth = { identityKey: `02${'11'.repeat(32)}`, userId: 7, isActive: true } +const abort = { reference: 'action-reference' } +const findArgs = { partial: {} } +const entityCases = [ + { + method: 'findOutputBasketsAuth', + params: [auth, findArgs], + run: (client: ForwardingClient) => client.findOutputBasketsAuth(auth, findArgs) + }, + { + method: 'findOutputsAuth', + params: [auth, findArgs], + run: (client: ForwardingClient) => client.findOutputsAuth(auth, findArgs) + }, + { + method: 'findProvenTxReqs', + params: [findArgs], + run: (client: ForwardingClient) => client.findProvenTxReqs(findArgs) + } +] +const cases = [ + ...entityCases, + { method: 'getCapabilities', params: [], run: (client: ForwardingClient) => client.getCapabilities() }, + { method: 'abortAction', params: [auth, abort], run: (client: ForwardingClient) => client.abortAction(auth, abort) }, + { + method: 'renewActionBatch', + params: [auth, 'batch-reference'], + run: (client: ForwardingClient) => client.renewActionBatch(auth, 'batch-reference') + } +] + +test.each(cases)('$method converts custom synchronous transport errors to rejected promises', async ({ run }) => { + const failure = new Error('synchronous custom transport failure') + const client = new ForwardingClient(() => { + throw failure + }) + const operation = run(client) + expect(operation).toBeInstanceOf(Promise) + await expect(operation).rejects.toBe(failure) +}) + +test.each(cases)('$method preserves asynchronous transport failure identity', async ({ run }) => { + const failure = new Error('asynchronous custom transport failure') + const client = new ForwardingClient(async () => { + throw failure + }) + await expect(run(client)).rejects.toBe(failure) +}) + +test.each(cases)( + '$method waits for transport settlement and retains parameters and result', + async ({ method, params, run }) => { + let finish!: (value: unknown) => void + const pending = new Promise(resolve => { + finish = resolve + }) + const send = jest.fn(() => pending) + const client = new ForwardingClient(send) + const settled = jest.fn() + const operation = run(client).then(value => { + settled() + return value + }) + await Promise.resolve() + expect(settled).not.toHaveBeenCalled() + expect(send).toHaveBeenCalledTimes(1) + expect(send).toHaveBeenCalledWith(method, params) + const result = { received: true } + finish(result) + await expect(operation).resolves.toBe(result) + expect(settled).toHaveBeenCalledTimes(1) + } +) + +test.each(entityCases)('$method normalizes dates, nulls and bytes in place', async ({ run }) => { + const record = { + created_at: '2024-01-01T00:00:00.000Z', + updated_at: 0, + optional: null, + bytes: new Uint8Array([0, 128, 255]) + } + const records = [record] + const client = new ForwardingClient(async () => records) + await expect(run(client)).resolves.toBe(records) + expect(records[0]).toBe(record) + expect(record.created_at).toEqual(new Date('2024-01-01T00:00:00.000Z')) + expect(record.updated_at).toEqual(new Date(0)) + expect(record.optional).toBeUndefined() + expect(record.bytes).toEqual([0, 128, 255]) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncTransfer.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncTransfer.test.ts index a9c5f3647..eb6b92af3 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncTransfer.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageClientBase.syncTransfer.test.ts @@ -37,10 +37,10 @@ const chunk = (): SyncChunk => ({ class TransferClient extends StorageClientBase { readonly request = jest.fn, [string, unknown[]]>() - constructor() { - super({} as WalletInterface, 'https://storage.example.test', { binaryRequests: true }) + constructor(binaryRequests = true, serverSupportsBinary = true) { + super({} as WalletInterface, 'https://storage.example.test', { binaryRequests }) Reflect.set(this, 'settings', { syncTransfer: capabilities }) - this.serverSupportsBinary = true + this.serverSupportsBinary = serverSupportsBinary } protected async rpcCall(method: string, params: unknown[]): Promise { return (await this.request(method, params)) as T @@ -137,3 +137,55 @@ test.each(['userIdentityKey', 'fromStorageIdentityKey', 'toStorageIdentityKey'] expect(client.request.mock.calls.at(-1)?.[0]).toBe('releaseSyncTransfer') } ) + +test('omitted inline capability retains the six MiB default without staging a small encoded page', async () => { + const client = new TransferClient() + Reflect.set(client, 'settings', { + syncTransfer: { version: 1, maxBytes: capabilities.maxBytes, partBytes: capabilities.partBytes } + }) + const result = { done: true, inserts: 1, updates: 0 } + client.request.mockImplementation(async method => { + expect(method).toBe('processSyncChunk') + return result + }) + await expect(client.processSyncChunk(args(), chunk())).resolves.toEqual(result) + expect(client.request).toHaveBeenCalledTimes(1) +}) + +test.each([ + { requestBinary: true, serverBinary: true }, + { requestBinary: false, serverBinary: true }, + { requestBinary: true, serverBinary: false }, + { requestBinary: false, serverBinary: false } +])('wire bytes require both binary negotiation flags %#', async ({ requestBinary, serverBinary }) => { + const client = new TransferClient(requestBinary, serverBinary), + page = chunk() + page.transactions![0].inputBEEF = Array(128).fill(173) + Reflect.set(client, 'settings', { syncTransfer: undefined }) + client.request.mockResolvedValue({ done: true, inserts: 1, updates: 0 }) + await client.processSyncChunk(args(), page) + const [method, params] = client.request.mock.calls[0], + wire = params[1] as SyncChunk + expect(method).toBe('processSyncChunk') + const binary = wire.transactions![0].inputBEEF + expect(Array.isArray(binary)).toBe(!(requestBinary && serverBinary)) + expect(Array.from(binary!)).toEqual(page.transactions![0].inputBEEF) +}) + +test('binary negotiation applies the one-byte expansion rather than legacy numeric-array overhead', async () => { + const client = new TransferClient(), + page = chunk() + const expectedPage = { + ...page, + transactions: [{ ...page.transactions![0], inputBEEF: Uint8Array.from(page.transactions![0].inputBEEF!) }] + } + const length = encodeSyncTransfer({ args: args(), chunk: expectedPage }).length + Reflect.set(client, 'settings', { syncTransfer: { ...capabilities, inlineBytes: 2 * length } }) + const result = { done: true, inserts: 1, updates: 0 } + client.request.mockImplementation(async method => { + expect(method).toBe('processSyncChunk') + return result + }) + await expect(client.processSyncChunk(args(), page)).resolves.toEqual(result) + expect(client.request).toHaveBeenCalledTimes(1) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts index f39dc4964..5e68a4ffd 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__test/StorageServerRpc.test.ts @@ -409,6 +409,32 @@ describe('StorageServer JSON-RPC boundary', () => { }) }) + test.each([ + 'readSnapshot', + 'supportsReadSnapshot', + 'openReadSnapshot', + 'supportsRetainedReadSnapshot', + 'openWalletReadSnapshot', + 'getSnapshotSync', + 'supportsWalletReadSnapshot' + ])( + 'keeps local snapshot method %s outside the authenticated RPC surface', + async method => { + const handler = jest.fn(async () => undefined) + const server = makeServer({ [method]: handler }) + const response = makeResponse() + await invoke( + server, + 'handleRpcRequest', + makeRequest({ jsonrpc: '2.0', method, params: [], id: 1 }), + response.response + ) + expect(response.statusCode).toBe(400) + expect(response.body).toMatchObject({ error: { code: -32601, message: `Method not found: ${method}` } }) + expect(handler).not.toHaveBeenCalled() + } + ) + test('redacts internal storage failures from JSON-RPC wallet errors', async () => { const server = makeServer( { @@ -918,3 +944,79 @@ describe('StorageServer JSON-RPC boundary', () => { ).rejects.toThrow('identityKey does not match authentication') }) }) + +describe('shared RPC framing used by snapshot transport', () => { + beforeEach(() => { + jest.spyOn(console, 'log').mockImplementation(() => undefined) + jest.spyOn(console, 'error').mockImplementation(() => undefined) + }) + afterEach(() => jest.restoreAllMocks()) + + test.each([{ jsonrpc: '1.0' }, { method: '' }, { method: 123 }, { params: {} }])( + 'rejects a malformed envelope at the RPC boundary: %p', + async changes => { + const server = makeServer() + const captured = makeResponse() + await invoke( + server, + 'handleRpcRequest', + makeRequest({ jsonrpc: '2.0', method: 'getSettings', params: [], id: 1, ...changes }), + captured.response + ) + expect(captured.statusCode).toBe(400) + expect(captured.body).toEqual({ error: { code: -32600, message: 'Invalid Request' } }) + expect(captured.headers[BINARY_ENCODING_HEADER]).toBeUndefined() + } + ) + + test.each([false, true])('decodes request bytes only with explicit negotiation: %p', async binary => { + const server = makeServer() + const dispatch = jest + .spyOn(server as never, 'dispatchRpcCall' as never) + .mockResolvedValue({ found: true, result: null } as never) + const captured = makeResponse() + const tagged = { $bsvBinary: 'base64', data: 'AQI=' } + await invoke( + server, + 'handleRpcRequest', + makeRequest( + { jsonrpc: '2.0', method: 'getSettings', params: [tagged], id: 1 }, + binary ? { [BINARY_REQUEST_ENCODING_HEADER]: BINARY_ENCODING } : {} + ), + captured.response + ) + expect(dispatch.mock.calls[0][1]).toEqual(binary ? [new Uint8Array([1, 2])] : [tagged]) + expect(captured.body).toEqual({ jsonrpc: '2.0', id: 1, result: null }) + }) + + test.each(['exact', 'over', 'unlimited'] as const)( + 'response limits preserve transfer negotiation at %s', + async mode => { + const chunk = { ...emptyChunk, padding: 'x'.repeat(1024) } + const bytes = Buffer.byteLength(JSON.stringify({ jsonrpc: '2.0', result: chunk, id: 1 })) + const maximum = mode === 'unlimited' ? -1 : bytes - Number(mode === 'over') + const server = makeServer({ getSyncChunk: async () => chunk }, { maxRpcResponseBytes: maximum }) + const beginRead = jest.fn(async () => ({ transferId: 'fixture-transfer' })) + Reflect.set(server, 'syncTransfers', { beginRead }) + const captured = makeResponse() + await invoke( + server, + 'handleRpcRequest', + makeRequest({ + jsonrpc: '2.0', + method: 'getSyncChunk', + params: [{ identityKey: 'alice', syncTransferVersion: 1 }], + id: 1 + }), + captured.response + ) + expect(captured.statusCode).toBe(200) + expect(beginRead).toHaveBeenCalledTimes(Number(mode === 'over')) + expect(captured.body).toEqual({ + jsonrpc: '2.0', + id: 1, + result: mode === 'over' ? { syncTransfer: { transferId: 'fixture-transfer' } } : chunk + }) + } + ) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts index c773dfb1a..fb6a316a4 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/__tests__/entityValidationHelpers.test.ts @@ -1,9 +1,4 @@ -import { - validateDate, - validateEntities, - validateEntity, - validateSyncChunkEntities -} from '../entityValidationHelpers' +import { validateDate, validateEntities, validateEntity, validateSyncChunkEntities } from '../entityValidationHelpers' import { EntityTimeStamp } from '../../../sdk/types' import { SyncChunk } from '../../../sdk/WalletStorage.interfaces' @@ -164,6 +159,34 @@ describe('entityValidationHelpers', () => { expect(result.id).toBe(1) }) + test('keeps data-only prototype and constructor keys as own values without changing the prototype', () => { + const entity = Object.assign(makeEntity(), JSON.parse('{"constructor":null}')) + Object.defineProperty(entity, '__proto__', { value: null, writable: true, enumerable: true, configurable: true }) + const prototype = Object.getPrototypeOf(entity) + validateEntity(entity) + expect(Object.getPrototypeOf(entity)).toBe(prototype) + for (const key of ['__proto__', 'constructor']) { + expect(Object.getOwnPropertyDescriptor(entity, key)).toEqual({ + value: undefined, + writable: true, + enumerable: true, + configurable: true + }) + } + }) + + test('preserves batch normalization of a non-enumerable own date field', () => { + const entity = makeEntity() + Object.defineProperty(entity, 'ts', { value: '2024-01-01T00:00:00.000Z', configurable: true }) + validateEntity(entity, ['ts']) + expect(Object.getOwnPropertyDescriptor(entity, 'ts')).toEqual({ + value: new Date('2024-01-01T00:00:00.000Z'), + writable: true, + enumerable: true, + configurable: true + }) + }) + test('only normalizes requested date fields that are own properties', () => { const e = makeEntity() const originalPrototype = Object.getPrototypeOf(e) @@ -199,7 +222,11 @@ describe('entityValidationHelpers', () => { test('validates every entity in a multi-entity array', () => { const arr: TestEntity[] = [ - { created_at: '2024-01-01T00:00:00.000Z' as unknown as Date, updated_at: '2024-01-01T00:00:00.000Z' as unknown as Date, name: null }, + { + created_at: '2024-01-01T00:00:00.000Z' as unknown as Date, + updated_at: '2024-01-01T00:00:00.000Z' as unknown as Date, + name: null + }, makeEntity({ blob: new Uint8Array([1, 2, 3]) }), makeEntity({ blob: Buffer.from([4, 5, 6]) as unknown as Uint8Array }) ] @@ -323,6 +350,15 @@ describe('entityValidationHelpers', () => { certificateFields: [makeEntity()] as never, user: makeEntity() as never } + const originalArrays = new Map() + for (const [name, value] of Object.entries(chunk)) { + if (!Array.isArray(value)) continue + const entities = value as TestEntity[] + originalArrays.set(name, entities) + entities[0].created_at = '2024-01-01T00:00:00.000Z' as unknown as Date + entities[0].updated_at = 0 as unknown as Date + entities[0].optional = null + } const result = validateSyncChunkEntities(chunk) expect(result).toBe(chunk) expect((result.provenTxs as unknown as TestEntity[])[0].blob).toEqual([1, 2]) @@ -345,7 +381,9 @@ describe('entityValidationHelpers', () => { ] for (const k of everyArrayKey) { const arr = result[k] as unknown as TestEntity[] + expect(arr).toBe(originalArrays.get(k)) expect(Array.isArray(arr)).toBe(true) + expect(arr[0].optional).toBeUndefined() expect(arr[0].created_at).toBeInstanceOf(Date) expect(arr[0].updated_at).toBeInstanceOf(Date) } diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts index ebe21dff3..1e9436427 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/entityValidationHelpers.ts @@ -16,23 +16,14 @@ export function validateDate(date: Date | string | number): Date { return new Date(date) } -function defineOwnValues(target: object, values: ReadonlyMap): void { - Object.defineProperties( - target, - Object.fromEntries( - Array.from(values, ([key, value]) => [ - key, - { - value, - writable: true, - enumerable: true, - configurable: true - } - ]) - ) - ) +function ownValue(value: unknown): PropertyDescriptor { + return { value, writable: true, enumerable: true, configurable: true } } +// Property definition copies descriptor fields; null normalization can share +// this immutable descriptor instead of allocating one for every nullable field. +const undefinedValue = Object.freeze(ownValue(undefined)) + /** * Force uniform behaviour across database engines. * Use to process all individual records with timestamps retrieved from database. @@ -41,24 +32,25 @@ export function validateEntity(entity: T, dateFields? const indexedEntity = entity as T & Record entity.created_at = validateDate(entity.created_at) entity.updated_at = validateDate(entity.updated_at) - const replacements = new Map() + const replacements = new Map() if (dateFields != null) { for (const df of dateFields) { const value = indexedEntity[df] if (Object.hasOwn(entity, df) && value) { - replacements.set(df, validateDate(value as Date | string | number)) + replacements.set(df, ownValue(validateDate(value as Date | string | number))) } } } for (const key of Object.keys(entity)) { - const val = replacements.has(key) ? replacements.get(key) : indexedEntity[key] + const replacement = replacements.get(key) + const val = replacement == null ? indexedEntity[key] : replacement.value if (val === null) { - replacements.set(key, undefined) + replacements.set(key, undefinedValue) } else if (val instanceof Uint8Array) { - replacements.set(key, Array.from(val)) + replacements.set(key, ownValue(Array.from(val))) } } - if (replacements.size > 0) defineOwnValues(entity, replacements) + Object.defineProperties(entity, Object.fromEntries(replacements)) return entity } @@ -116,18 +108,10 @@ function validateSyncChunkTotals(totals: SyncChunkTotals): void { */ export function validateSyncChunkEntities(r: SyncChunk): SyncChunk { if (r.totals != null) validateSyncChunkTotals(r.totals) - if (r.certificateFields != null) r.certificateFields = validateEntities(r.certificateFields) - if (r.certificates != null) r.certificates = validateEntities(r.certificates) - if (r.commissions != null) r.commissions = validateEntities(r.commissions) - if (r.outputBaskets != null) r.outputBaskets = validateEntities(r.outputBaskets) - if (r.outputTagMaps != null) r.outputTagMaps = validateEntities(r.outputTagMaps) - if (r.outputTags != null) r.outputTags = validateEntities(r.outputTags) - if (r.outputs != null) r.outputs = validateEntities(r.outputs) - if (r.provenTxReqs != null) r.provenTxReqs = validateEntities(r.provenTxReqs) - if (r.provenTxs != null) r.provenTxs = validateEntities(r.provenTxs) - if (r.transactions != null) r.transactions = validateEntities(r.transactions) - if (r.txLabelMaps != null) r.txLabelMaps = validateEntities(r.txLabelMaps) - if (r.txLabels != null) r.txLabels = validateEntities(r.txLabels) + for (const name of syncChunkTotalRecordNames) { + const entities = r[name] + if (entities != null) validateEntities(entities) + } if (r.user != null) r.user = validateEntity(r.user) return r } diff --git a/packages/wallet/wallet-toolbox/src/storage/remoting/validateRpcSyncProofs.ts b/packages/wallet/wallet-toolbox/src/storage/remoting/validateRpcSyncProofs.ts index 98f19d9ed..ee84aae90 100644 --- a/packages/wallet/wallet-toolbox/src/storage/remoting/validateRpcSyncProofs.ts +++ b/packages/wallet/wallet-toolbox/src/storage/remoting/validateRpcSyncProofs.ts @@ -1,50 +1,13 @@ -import { WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' -import type { GetMerklePathResult } from '../../sdk/WalletServices.interfaces' -import { doubleSha256BE } from '../../utility/utilityHelpers' -import { asString } from '../../utility/utilityHelpers.noBuffer' -import { StaleSyncProofError, validateSyncProof, markSyncProofReconciled } from '../methods/validateSyncProof' +import { StaleSyncProofError, validateSyncProof } from '../methods/validateSyncProof' import type { SyncProofValidationStorage } from '../methods/validateSyncProof' import type { TableProvenTx } from '../schema/tables/TableProvenTx' - -/** Obtain replacement metadata; never alter transaction bytes, IDs or timestamps. */ -async function refreshSyncProof(storage: SyncProofValidationStorage, candidate: TableProvenTx): Promise { - const services = storage.getServices() - let current: TableProvenTx | undefined - const validate = async ({ merklePath }: GetMerklePathResult): Promise => { - if (merklePath == null) throw new Error('Current proof unavailable') - const header = await services.getHeaderForHeight(merklePath.blockHeight) - const replacement = { - ...candidate, - height: merklePath.blockHeight, - index: merklePath.path[0]?.find(leaf => leaf.txid === true && leaf.hash === candidate.txid)?.offset ?? -1, - merklePath: merklePath.toBinary(), - merkleRoot: merklePath.computeRoot(candidate.txid), - blockHash: asString(doubleSha256BE(header)) - } - // Recheck raw bytes, membership, root and active header before authorizing - // any global proof replacement. A provider's success response is insufficient. - await validateSyncProof(storage, replacement) - markSyncProofReconciled(replacement) - current = replacement - } - try { - if (services.getValidatedMerklePath != null) { - await services.getValidatedMerklePath(candidate.txid, validate) - } else { - // Custom services remain compatible; one bounded lookup, fully validated. - await validate(await services.getMerklePath(candidate.txid)) - } - if (current != null) return current - } catch { - // Report a recoverable sync error, without leaking provider response data. - } - throw new WERR_INVALID_PARAMETER('provenTx', - 'a server-verified proof. Merkle root is not active at the recorded height and a current proof could not be verified. ' + - 'Ask the source provider to reconcile this transaction, then resume synchronization; saved wallet data is unchanged.') -} +import { refreshSyncProof } from '../methods/refreshSyncProof' /** Validate a whole RPC page with at most eight proofs in flight and no database writes. */ -export async function validateSyncProofs(storage: SyncProofValidationStorage, candidates: TableProvenTx[]): Promise { +export async function validateSyncProofs( + storage: SyncProofValidationStorage, + candidates: TableProvenTx[] +): Promise { let next = 0 let failed = false const validated: TableProvenTx[] = [] diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts index 380e92fc0..01a09c6f7 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/KnexMigrations.ts @@ -1,5 +1,49 @@ +import { forSnapshotSqlDialect } from './snapshotSqlMigration' +import { migrateGeneration, refuseGenerationDowngrade } from './snapshotSqliteIndexMigration' +import { migration as SNAPSHOT_SQLITE_INDEX_MIGRATION } from './snapshotSqliteIndexState' +import { + addSnapshotGlobalIndexes, + removeSnapshotGlobalIndexes, + SNAPSHOT_GLOBAL_INDEX_MIGRATION +} from './snapshotGlobalIndexMigration' +import { + addSnapshotCertificateIndexes, + removeSnapshotCertificateIndexes, + SNAPSHOT_CERTIFICATE_INDEX_MIGRATION +} from './snapshotCertificateIndexMigration' +import { + addSnapshotRelationIndexes, + removeSnapshotRelationIndexes, + SNAPSHOT_RELATION_INDEX_MIGRATION +} from './snapshotRelationIndexMigration' +import { + addSnapshotProfileIndexes, + removeSnapshotProfileIndexes, + SNAPSHOT_PROFILE_INDEX_MIGRATION +} from './snapshotProfileIndexMigration' +import { + addSnapshotArchiveGuardTable, + removeSnapshotArchiveGuardTable, + SNAPSHOT_ARCHIVE_GUARD_MIGRATION +} from './snapshotArchiveGuardMigration' +import { + addSnapshotArchiveOwnerTable, + removeSnapshotArchiveOwnerTable, + SNAPSHOT_ARCHIVE_OWNER_MIGRATION +} from './snapshotArchiveOwnerMigration' +import { + addSnapshotArchiveRequestTable, + removeSnapshotArchiveRequestTable, + SNAPSHOT_ARCHIVE_REQUEST_MIGRATION +} from './snapshotArchiveRequestMigration' +import { + addSnapshotArchiveTables, + removeSnapshotArchiveTables, + SNAPSHOT_ARCHIVE_MIGRATION +} from './snapshotArchiveMigration' /* eslint-disable @typescript-eslint/no-unused-vars */ import { Knex } from 'knex' +import { addSnapshotSyncTables, removeSnapshotSyncTables, SNAPSHOT_SYNC_MIGRATION } from './snapshotSyncMigration' import { DBType } from '../StorageReader' import { Chain } from '../../sdk/types' import { StorageKnex } from '../StorageKnex' @@ -11,6 +55,17 @@ import { LEGACY_MANAGED_CHANGE_MINIMUM_SATOSHIS } from '../methods/managedChangePolicy' +export { SNAPSHOT_GLOBAL_INDEX_MIGRATION } from './snapshotGlobalIndexMigration' +export { migration as SNAPSHOT_SQLITE_INDEX_MIGRATION } from './snapshotSqliteIndexState' +export { SNAPSHOT_CERTIFICATE_INDEX_MIGRATION } from './snapshotCertificateIndexMigration' +export { SNAPSHOT_RELATION_INDEX_MIGRATION } from './snapshotRelationIndexMigration' +export { SNAPSHOT_PROFILE_INDEX_MIGRATION } from './snapshotProfileIndexMigration' +export { SNAPSHOT_ARCHIVE_OWNER_MIGRATION } from './snapshotArchiveOwnerMigration' +export { SNAPSHOT_ARCHIVE_GUARD_MIGRATION } from './snapshotArchiveGuardMigration' +export { SNAPSHOT_ARCHIVE_REQUEST_MIGRATION } from './snapshotArchiveRequestMigration' +export { SNAPSHOT_ARCHIVE_MIGRATION } from './snapshotArchiveMigration' +export { SNAPSHOT_SYNC_MIGRATION } from './snapshotSyncMigration' + export const SYNC_TRANSFER_MIGRATION = '2026-09-09-001 add bounded sync transfers' export const AUTH_SESSION_MIGRATION = '2026-07-14-001 add shared auth sessions' @@ -93,6 +148,68 @@ export class KnexMigrations implements MigrationSource { } } + // DDL may commit independently on MySQL. Bootstrap pages retain their own + // durable positions on both backends and resume before journal publication. + migrations[SNAPSHOT_SQLITE_INDEX_MIGRATION] = { + config: { transaction: false }, + up: forSnapshotSqlDialect(migrateGeneration), + down: forSnapshotSqlDialect(refuseGenerationDowngrade) + } + + migrations[SNAPSHOT_GLOBAL_INDEX_MIGRATION] = { + config: { transaction: false }, + up: forSnapshotSqlDialect(addSnapshotGlobalIndexes), + down: forSnapshotSqlDialect(removeSnapshotGlobalIndexes) + } + + migrations[SNAPSHOT_CERTIFICATE_INDEX_MIGRATION] = { + config: { transaction: false }, + up: forSnapshotSqlDialect(addSnapshotCertificateIndexes), + down: forSnapshotSqlDialect(removeSnapshotCertificateIndexes) + } + + migrations[SNAPSHOT_RELATION_INDEX_MIGRATION] = { + config: { transaction: false }, + up: forSnapshotSqlDialect(addSnapshotRelationIndexes), + down: forSnapshotSqlDialect(removeSnapshotRelationIndexes) + } + + migrations[SNAPSHOT_PROFILE_INDEX_MIGRATION] = { + config: { transaction: false }, + up: forSnapshotSqlDialect(addSnapshotProfileIndexes), + down: forSnapshotSqlDialect(removeSnapshotProfileIndexes) + } + + migrations[SNAPSHOT_ARCHIVE_GUARD_MIGRATION] = { + config: { transaction: true }, + up: forSnapshotSqlDialect(addSnapshotArchiveGuardTable), + down: forSnapshotSqlDialect(removeSnapshotArchiveGuardTable) + } + + migrations[SNAPSHOT_ARCHIVE_OWNER_MIGRATION] = { + config: { transaction: true }, + up: forSnapshotSqlDialect(addSnapshotArchiveOwnerTable), + down: forSnapshotSqlDialect(removeSnapshotArchiveOwnerTable) + } + + migrations[SNAPSHOT_ARCHIVE_REQUEST_MIGRATION] = { + config: { transaction: true }, + up: forSnapshotSqlDialect(addSnapshotArchiveRequestTable), + down: forSnapshotSqlDialect(removeSnapshotArchiveRequestTable) + } + + migrations[SNAPSHOT_ARCHIVE_MIGRATION] = { + config: { transaction: true }, + up: forSnapshotSqlDialect(addSnapshotArchiveTables), + down: forSnapshotSqlDialect(removeSnapshotArchiveTables) + } + + migrations[SNAPSHOT_SYNC_MIGRATION] = { + config: { transaction: true }, + up: forSnapshotSqlDialect(addSnapshotSyncTables), + down: forSnapshotSqlDialect(removeSnapshotSyncTables) + } + migrations[SYNC_TRANSFER_MIGRATION] = { config: { transaction: true }, async up(knex) { diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTxReq.ts b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTxReq.ts index 6ccdcdc36..533e6f288 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTxReq.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntityProvenTxReq.ts @@ -26,11 +26,7 @@ export class EntityProvenTxReq extends EntityBase { return new EntityProvenTxReq(reqApi) } - static fromTxid( - txid: string, - rawTx: number[] | Uint8Array, - inputBEEF?: number[] | Uint8Array - ): EntityProvenTxReq { + static fromTxid(txid: string, rawTx: number[] | Uint8Array, inputBEEF?: number[] | Uint8Array): EntityProvenTxReq { const now = new Date() return new EntityProvenTxReq({ provenTxReqId: 0, @@ -626,6 +622,10 @@ export class EntityProvenTxReq extends EntityBase { syncMap: SyncMap, trx?: TrxToken ): Promise { + const previousBatch = this.batch + const previousHistory = this.apiHistory + const previousNotify = this.apiNotify + const previousUpdatedAt = this.updated_at.getTime() if (!this.batch && ei.batch) this.batch = ei.batch else if (this.batch && ei.batch && this.batch !== ei.batch) { throw new WERR_INTERNAL('ProvenTxReq merge batch not equal.') @@ -635,6 +635,13 @@ export class EntityProvenTxReq extends EntityBase { this.mergeNotifyTransactionIds(ei, syncMap) this.updated_at = new Date(Math.max(ei.updated_at.getTime(), this.updated_at.getTime())) + if ( + this.batch === previousBatch && + this.apiHistory === previousHistory && + this.apiNotify === previousNotify && + this.updated_at.getTime() === previousUpdatedAt + ) + return false await storage.updateProvenTxReq(this.id, this.toApi(), trx) return false } diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts index dd8cd13fb..734d093af 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/entities/EntitySyncState.ts @@ -1,3 +1,4 @@ +import { mergeSyncChunkEntities } from './mergeSyncChunkEntities' import { RequestSyncChunkArgs, SyncChunk, @@ -7,24 +8,10 @@ import { WalletStorageSync } from '../../../sdk/WalletStorage.interfaces' import { WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' -import { maxDate, verifyId, verifyTruthy } from '../../../utility/utilityHelpers' +import { verifyId, verifyTruthy } from '../../../utility/utilityHelpers' import { TableSettings } from '../tables/TableSettings' import { TableSyncState } from '../tables/TableSyncState' import { createSyncMap, EntityBase, EntityStorage, SyncError, SyncMap } from './EntityBase' -import { EntityCertificate } from './EntityCertificate' -import { EntityCertificateField } from './EntityCertificateField' -import { EntityCommission } from './EntityCommission' -import { EntityOutput } from './EntityOutput' -import { EntityOutputBasket } from './EntityOutputBasket' -import { EntityOutputTag } from './EntityOutputTag' -import { EntityOutputTagMap } from './EntityOutputTagMap' -import { EntityProvenTx } from './EntityProvenTx' -import { EntityProvenTxReq } from './EntityProvenTxReq' -import { EntityTransaction } from './EntityTransaction' -import { EntityTxLabel } from './EntityTxLabel' -import { EntityTxLabelMap } from './EntityTxLabelMap' -import { EntityUser } from './EntityUser' -import { MergeEntity } from './MergeEntity' function formatSyncSection( heading: string, @@ -312,7 +299,7 @@ export class EntitySyncState extends EntityBase { maxRoughSize: maxRoughSize || 10000000, maxItems: maxItems || 1000, offsets: [], - since: this.when, + since: this.when == null ? undefined : new Date(this.when), fromStorageIdentityKey: this.storageIdentityKey, toStorageIdentityKey: forStorageIdentityKey } @@ -338,7 +325,11 @@ export class EntitySyncState extends EntityBase { /** Return progress without the potentially large writer-local ID maps. */ makeSyncCheckpoint(): SyncCheckpoint { const request = this.makeRequestSyncChunkArgs('', '') - return { syncStateId: this.id, since: this.when == null ? undefined : new Date(this.when), offsets: request.offsets } + return { + syncStateId: this.id, + since: this.when == null ? undefined : new Date(this.when), + offsets: request.offsets + } } static syncChunkSummary(c: SyncChunk): string { @@ -378,54 +369,16 @@ export class EntitySyncState extends EntityBase { updates: number inserts: number }> { - const mes = [ - new MergeEntity(chunk.provenTxs, EntityProvenTx.mergeFind, this.syncMap.provenTx), - new MergeEntity(chunk.outputBaskets, EntityOutputBasket.mergeFind, this.syncMap.outputBasket), - new MergeEntity(chunk.outputTags, EntityOutputTag.mergeFind, this.syncMap.outputTag), - new MergeEntity(chunk.txLabels, EntityTxLabel.mergeFind, this.syncMap.txLabel), - new MergeEntity(chunk.transactions, EntityTransaction.mergeFind, this.syncMap.transaction), - new MergeEntity(chunk.outputs, EntityOutput.mergeFind, this.syncMap.output), - new MergeEntity(chunk.txLabelMaps, EntityTxLabelMap.mergeFind, this.syncMap.txLabelMap), - new MergeEntity(chunk.outputTagMaps, EntityOutputTagMap.mergeFind, this.syncMap.outputTagMap), - new MergeEntity(chunk.certificates, EntityCertificate.mergeFind, this.syncMap.certificate), - new MergeEntity(chunk.certificateFields, EntityCertificateField.mergeFind, this.syncMap.certificateField), - new MergeEntity(chunk.commissions, EntityCommission.mergeFind, this.syncMap.commission), - new MergeEntity(chunk.provenTxReqs, EntityProvenTxReq.mergeFind, this.syncMap.provenTxReq) - ] - - let updates = 0 - let inserts = 0 - let maxUpdated_at: Date | undefined - let done = true - - // Merge User - if (chunk.user != null) { - const ei = chunk.user - const { found, eo } = await EntityUser.mergeFind(writer, this.userId, ei, trx) - if (found) { - if (await eo.mergeExisting(writer, args.since, ei, undefined, trx)) { - maxUpdated_at = maxDate(maxUpdated_at, ei.updated_at) - updates++ - } - } - } - - // Merge everything else... - for (const me of mes) { - const r = await me.merge(args.since, writer, this.userId, this.syncMap, trx) - // The counts become the offsets for the next chunk. - me.esm.count += me.stateArray?.length || 0 - updates += r.updates - inserts += r.inserts - maxUpdated_at = maxDate(maxUpdated_at, me.esm.maxUpdated_at) - // If any entity type either did not report results or if there were at least one, then we aren't done. - if (me.stateArray === undefined || me.stateArray.length > 0) done = false - } - + const { done, maxUpdated_at, updates, inserts } = await mergeSyncChunkEntities( + writer, + this.userId, + args.since, + chunk, + this.syncMap, + trx + ) if (done) { - // Next batch starts further in the future with offsets of zero. this.when = maxUpdated_at - for (const me of mes) me.esm.count = 0 } await this.updateStorage(writer, false, trx) diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/entities/__tests/ProvenTxTests.test.ts b/packages/wallet/wallet-toolbox/src/storage/schema/entities/__tests/ProvenTxTests.test.ts index 99ef6e120..f761f3c50 100644 --- a/packages/wallet/wallet-toolbox/src/storage/schema/entities/__tests/ProvenTxTests.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/schema/entities/__tests/ProvenTxTests.test.ts @@ -14,7 +14,7 @@ function makeServerVerifiedProof( height: number, time: number, provenTxId = 1 -): { proof: TableProvenTx, header: number[] } { +): { proof: TableProvenTx; header: number[] } { const txid = transaction.id('hex') const merklePath = new bsv.MerklePath(height, [[{ offset: 0, hash: txid, txid: true }]]) const merkleRoot = merklePath.computeRoot(txid) @@ -49,7 +49,7 @@ function makeProofStorage(header: number[]): StorageProvider & { updateProvenTx: jest.Mock invalidatePreparedBeefs: jest.Mock } { - return { + return Object.assign(Object.create(StorageProvider.prototype), { getServices: () => ({ getChainTracker: async () => ({ isValidRootForHeight: async () => true }), getHeaderForHeight: async () => header @@ -57,7 +57,7 @@ function makeProofStorage(header: number[]): StorageProvider & { insertProvenTx: jest.fn(async () => 77), updateProvenTx: jest.fn(async () => 1), invalidatePreparedBeefs: jest.fn(async () => 0) - } as unknown as StorageProvider & { + }) as StorageProvider & { insertProvenTx: jest.Mock updateProvenTx: jest.Mock invalidatePreparedBeefs: jest.Mock @@ -570,8 +570,9 @@ describe('ProvenTx class method tests', () => { const corrupted = { ...proof, rawTx: [...proof.rawTx] } corrupted.rawTx[corrupted.rawTx.length - 1] ^= 1 - await expect(syncProofValidation.validateSyncProof(storage, corrupted)) - .rejects.toMatchObject({ code: 'WERR_INVALID_PARAMETER' }) + await expect(syncProofValidation.validateSyncProof(storage, corrupted)).rejects.toMatchObject({ + code: 'WERR_INVALID_PARAMETER' + }) expect(storage.insertProvenTx).not.toHaveBeenCalled() expect(storage.invalidatePreparedBeefs).not.toHaveBeenCalled() @@ -600,17 +601,16 @@ describe('ProvenTx class method tests', () => { const entity = new EntityProvenTx(original.proof) await syncProofValidation.validateSyncProof(storage, replacement.proof) - await expect(entity.mergeExisting( - storage, - undefined, - replacement.proof, - createSyncMap() - )).resolves.toBe(true) - - expect(storage.updateProvenTx).toHaveBeenCalledWith(44, expect.objectContaining({ - height: 103, - blockHash: replacement.proof.blockHash - }), undefined) + await expect(entity.mergeExisting(storage, undefined, replacement.proof, createSyncMap())).resolves.toBe(true) + + expect(storage.updateProvenTx).toHaveBeenCalledWith( + 44, + expect.objectContaining({ + height: 103, + blockHash: replacement.proof.blockHash + }), + undefined + ) expect(storage.invalidatePreparedBeefs).toHaveBeenCalledWith(undefined) }) @@ -626,11 +626,13 @@ describe('ProvenTx class method tests', () => { storage.transaction = jest.fn() storage.findProvenTxs = jest.fn(async () => [active.proof]) - await expect(StorageProvider.prototype.processSyncChunk.call( - storage, - {} as sdk.RequestSyncChunkArgs, - { provenTxs: [stale.proof] } as sdk.SyncChunk - )).rejects.toMatchObject({ code: 'WERR_INVALID_PARAMETER' }) + await expect( + StorageProvider.prototype.processSyncChunk.call( + storage, + {} as sdk.RequestSyncChunkArgs, + { provenTxs: [stale.proof] } as sdk.SyncChunk + ) + ).rejects.toMatchObject({ code: 'WERR_INVALID_PARAMETER' }) expect(storage.transaction).not.toHaveBeenCalled() }) @@ -650,11 +652,13 @@ describe('ProvenTx class method tests', () => { storage.transaction = jest.fn(async () => merged) storage.findProvenTxs = jest.fn(async () => []) - await expect(StorageProvider.prototype.processSyncChunk.call( - storage, - {} as sdk.RequestSyncChunkArgs, - { provenTxs: [proof] } as sdk.SyncChunk - )).resolves.toEqual(merged) + await expect( + StorageProvider.prototype.processSyncChunk.call( + storage, + {} as sdk.RequestSyncChunkArgs, + { provenTxs: [proof] } as sdk.SyncChunk + ) + ).resolves.toEqual(merged) expect(proof).toMatchObject({ txid: proof.txid.toLowerCase(), @@ -672,12 +676,9 @@ describe('ProvenTx class method tests', () => { const storage = makeProofStorage(unvalidated.header) const entity = new EntityProvenTx(original.proof) - await expect(entity.mergeExisting( - storage, - undefined, - unvalidated.proof, - createSyncMap() - )).rejects.toMatchObject({ code: 'WERR_INVALID_PARAMETER' }) + await expect(entity.mergeExisting(storage, undefined, unvalidated.proof, createSyncMap())).rejects.toMatchObject({ + code: 'WERR_INVALID_PARAMETER' + }) expect(storage.updateProvenTx).not.toHaveBeenCalled() }) @@ -698,8 +699,9 @@ describe('ProvenTx class method tests', () => { ] for (const candidate of malformed) { - await expect(syncProofValidation.validateSyncProof(storage, candidate)) - .rejects.toMatchObject({ code: 'WERR_INVALID_PARAMETER' }) + await expect(syncProofValidation.validateSyncProof(storage, candidate)).rejects.toMatchObject({ + code: 'WERR_INVALID_PARAMETER' + }) } }) @@ -707,11 +709,8 @@ describe('ProvenTx class method tests', () => { const transaction = new bsv.Transaction() transaction.addOutput({ satoshis: 1, lockingScript: bsv.Script.fromHex('51') }) const { proof, header } = makeServerVerifiedProof(transaction, 110, 11) - const wrongHeightPath = new bsv.MerklePath( - proof.height + 1, - [[{ offset: 0, hash: proof.txid, txid: true }]] - ) - const invalidCases: Array<{ candidate: TableProvenTx, storage: StorageProvider }> = [ + const wrongHeightPath = new bsv.MerklePath(proof.height + 1, [[{ offset: 0, hash: proof.txid, txid: true }]]) + const invalidCases: Array<{ candidate: TableProvenTx; storage: StorageProvider }> = [ { candidate: { ...proof, merklePath: wrongHeightPath.toBinary() }, storage: makeProofStorage(header) }, { candidate: { ...proof, index: 1 }, storage: makeProofStorage(header) }, { candidate: { ...proof, merkleRoot: 'f'.repeat(64) }, storage: makeProofStorage(header) }, @@ -728,7 +727,9 @@ describe('ProvenTx class method tests', () => { candidate: { ...proof }, storage: { getServices: () => ({ - getChainTracker: async () => { throw new Error('chain tracker unavailable') }, + getChainTracker: async () => { + throw new Error('chain tracker unavailable') + }, getHeaderForHeight: async () => header }) } as unknown as StorageProvider @@ -738,8 +739,9 @@ describe('ProvenTx class method tests', () => { ] for (const { candidate, storage } of invalidCases) { - await expect(syncProofValidation.validateSyncProof(storage, candidate)) - .rejects.toMatchObject({ code: 'WERR_INVALID_PARAMETER' }) + await expect(syncProofValidation.validateSyncProof(storage, candidate)).rejects.toMatchObject({ + code: 'WERR_INVALID_PARAMETER' + }) } }) @@ -750,8 +752,8 @@ describe('ProvenTx class method tests', () => { syncProofValidation.markSyncProofInsertOnly(proof) - expect(() => syncProofValidation.assertSyncProofReplacementAuthorized(proof)) - .toThrow('concurrent proof row appeared') + expect(() => syncProofValidation.assertSyncProofReplacementAuthorized(proof)).toThrow( + 'concurrent proof row appeared' + ) }) - }) diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/entities/mergeSyncChunkEntities.ts b/packages/wallet/wallet-toolbox/src/storage/schema/entities/mergeSyncChunkEntities.ts new file mode 100644 index 000000000..05261d42f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/entities/mergeSyncChunkEntities.ts @@ -0,0 +1,75 @@ +import type { SyncChunk, TrxToken } from '../../../sdk/WalletStorage.interfaces' +import { maxDate } from '../../../utility/utilityHelpers' +import { runInSeries } from '../../../utility/runInSeries' +import type { EntityStorage, SyncMap } from './EntityBase' +import { EntityCertificate } from './EntityCertificate' +import { EntityCertificateField } from './EntityCertificateField' +import { EntityCommission } from './EntityCommission' +import { EntityOutput } from './EntityOutput' +import { EntityOutputBasket } from './EntityOutputBasket' +import { EntityOutputTag } from './EntityOutputTag' +import { EntityOutputTagMap } from './EntityOutputTagMap' +import { EntityProvenTx } from './EntityProvenTx' +import { EntityProvenTxReq } from './EntityProvenTxReq' +import { EntityTransaction } from './EntityTransaction' +import { EntityTxLabel } from './EntityTxLabel' +import { EntityTxLabelMap } from './EntityTxLabelMap' +import { EntityUser } from './EntityUser' +import { MergeEntity } from './MergeEntity' + +/** Merge a bounded set of rows using a caller-owned ID map and transaction. No checkpoint is persisted here. */ +export async function mergeSyncChunkEntities( + writer: EntityStorage, + userId: number, + since: Date | undefined, + chunk: SyncChunk, + syncMap: SyncMap, + trx?: TrxToken +): Promise<{ done: boolean; maxUpdated_at: Date | undefined; updates: number; inserts: number }> { + const mes = [ + new MergeEntity(chunk.provenTxs, EntityProvenTx.mergeFind, syncMap.provenTx), + new MergeEntity(chunk.outputBaskets, EntityOutputBasket.mergeFind, syncMap.outputBasket), + new MergeEntity(chunk.outputTags, EntityOutputTag.mergeFind, syncMap.outputTag), + new MergeEntity(chunk.txLabels, EntityTxLabel.mergeFind, syncMap.txLabel), + new MergeEntity(chunk.transactions, EntityTransaction.mergeFind, syncMap.transaction), + new MergeEntity(chunk.outputs, EntityOutput.mergeFind, syncMap.output), + new MergeEntity(chunk.txLabelMaps, EntityTxLabelMap.mergeFind, syncMap.txLabelMap), + new MergeEntity(chunk.outputTagMaps, EntityOutputTagMap.mergeFind, syncMap.outputTagMap), + new MergeEntity(chunk.certificates, EntityCertificate.mergeFind, syncMap.certificate), + new MergeEntity(chunk.certificateFields, EntityCertificateField.mergeFind, syncMap.certificateField), + new MergeEntity(chunk.commissions, EntityCommission.mergeFind, syncMap.commission), + new MergeEntity(chunk.provenTxReqs, EntityProvenTxReq.mergeFind, syncMap.provenTxReq) + ] + + let updates = 0 + let inserts = 0 + let maxUpdated_at: Date | undefined + let done = true + + // Merge User + if (chunk.user != null) { + const ei = chunk.user + const { found, eo } = await EntityUser.mergeFind(writer, userId, ei, trx) + if (found) { + if (await eo.mergeExisting(writer, since, ei, undefined, trx)) { + maxUpdated_at = maxDate(maxUpdated_at, ei.updated_at) + updates++ + } + } + } + + // Child entities depend on ID mappings committed by their predecessors. + await runInSeries(mes, async me => { + const r = await me.merge(since, writer, userId, syncMap, trx) + // The counts become the offsets for the next chunk. + me.esm.count += me.stateArray?.length || 0 + updates += r.updates + inserts += r.inserts + maxUpdated_at = maxDate(maxUpdated_at, me.esm.maxUpdated_at) + // If any entity type either did not report results or if there were at least one, then we aren't done. + if (me.stateArray === undefined || me.stateArray.length > 0) done = false + }) + + if (done) for (const me of mes) me.esm.count = 0 + return { done, maxUpdated_at, updates, inserts } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveGuardMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveGuardMigration.ts new file mode 100644 index 000000000..4621465a0 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveGuardMigration.ts @@ -0,0 +1,34 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { snapshotArchiveLimits } from '../snapshot/archive/SnapshotArchive' + +export const SNAPSHOT_ARCHIVE_GUARD_MIGRATION = '2026-10-01-002 add snapshot archive source guards' + +/** Older candidate owners default to zero and can never be inferred to use a guard. */ +export async function addSnapshotArchiveGuardTable(knex: Knex): Promise { + if (!(await knex.schema.hasColumn('snapshot_archive_owners', 'guardVersion'))) { + await knex.schema.alterTable('snapshot_archive_owners', table => { + table.integer('guardVersion').notNullable().defaultTo(0) + }) + } + if (!(await knex.schema.hasTable('snapshot_archive_owner_slots'))) { + await knex.schema.createTable('snapshot_archive_owner_slots', table => { + table.integer('slot').primary() + table.text('bindingJson').nullable() + }) + } + await knex('snapshot_archive_owner_slots') + .insert(Array.from({ length: snapshotArchiveLimits.archives }, (_, slot) => ({ slot, bindingJson: null }))) + .onConflict('slot') + .ignore() +} + +export async function removeSnapshotArchiveGuardTable(knex: Knex): Promise { + if ((await knex('snapshot_archive_owners').first('slot')) !== undefined) { + throw new WERR_INVALID_OPERATION('Drain snapshot archive sources before removing their guards') + } + await knex.schema.dropTableIfExists('snapshot_archive_owner_slots') + if (await knex.schema.hasColumn('snapshot_archive_owners', 'guardVersion')) { + await knex.schema.alterTable('snapshot_archive_owners', table => table.dropColumn('guardVersion')) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveMigration.ts new file mode 100644 index 000000000..75236d674 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveMigration.ts @@ -0,0 +1,50 @@ +import type { Knex } from 'knex' + +export const SNAPSHOT_ARCHIVE_MIGRATION = '2026-09-30-002 add snapshot archive staging' + +/** Separate from standard wallet tables and legacy synchronization checkpoints. */ +export async function addSnapshotArchiveTables(knex: Knex): Promise { + if (!(await knex.schema.hasTable('snapshot_archive_capacity'))) { + await knex.schema.createTable('snapshot_archive_capacity', table => { + table.integer('id').primary() + table.integer('archives').notNullable() + table.bigInteger('reservedBytes').notNullable() + }) + } + await knex('snapshot_archive_capacity').insert({ id: 1, archives: 0, reservedBytes: 0 }).onConflict('id').ignore() + if (!(await knex.schema.hasTable('snapshot_archives'))) { + await knex.schema.createTable('snapshot_archives', table => { + table.string('archiveId', 64).primary() + table.string('identityKey', 130).notNullable().unique() + table.string('writerToken', 64).notNullable() + table.string('state', 16).notNullable() + table.text('binding', 'mediumtext').notNullable() + table.bigInteger('expiresAt').notNullable().index() + table.bigInteger('reservedBytes').notNullable() + table.bigInteger('usedBytes').notNullable() + table.integer('nextSequence').notNullable() + table.integer('tableIndex').notNullable() + table.bigInteger('rows').notNullable() + table.string('digest', 64).notNullable() + }) + } + if (!(await knex.schema.hasTable('snapshot_archive_pages'))) { + await knex.schema.createTable('snapshot_archive_pages', table => { + table.string('archiveId', 64).notNullable() + table.integer('sequence').notNullable() + table.string('tableName', 32).notNullable() + table.integer('rows').notNullable() + table.boolean('done').notNullable() + table.string('digest', 64).notNullable() + const mysql = String(knex.client.config.client).includes('mysql') + table.specificType('payload', mysql ? 'MEDIUMBLOB' : 'BLOB').notNullable() + table.primary(['archiveId', 'sequence']) + }) + } +} + +export async function removeSnapshotArchiveTables(knex: Knex): Promise { + await knex.schema.dropTableIfExists('snapshot_archive_pages') + await knex.schema.dropTableIfExists('snapshot_archives') + await knex.schema.dropTableIfExists('snapshot_archive_capacity') +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveOwnerMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveOwnerMigration.ts new file mode 100644 index 000000000..296b2d273 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveOwnerMigration.ts @@ -0,0 +1,26 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' + +export const SNAPSHOT_ARCHIVE_OWNER_MIGRATION = '2026-10-01-001 add snapshot archive source owners' + +export async function addSnapshotArchiveOwnerTable(knex: Knex): Promise { + if (!(await knex.schema.hasTable('snapshot_archive_owners'))) { + await knex.schema.createTable('snapshot_archive_owners', table => { + table.integer('slot').primary() + table.string('identityKey', 130).notNullable() + table.string('requestId', 64).notNullable() + table.string('claimToken', 64).notNullable() + table.string('archiveId', 64).nullable().unique() + table.unique(['identityKey', 'requestId']) + }) + } +} + +export async function removeSnapshotArchiveOwnerTable(knex: Knex): Promise { + if (await knex.schema.hasTable('snapshot_archive_owners')) { + if ((await knex('snapshot_archive_owners').first('slot')) !== undefined) { + throw new WERR_INVALID_OPERATION('Drain snapshot archive sources before removing their schema') + } + } + await knex.schema.dropTableIfExists('snapshot_archive_owners') +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveRequestMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveRequestMigration.ts new file mode 100644 index 000000000..767b51bce --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotArchiveRequestMigration.ts @@ -0,0 +1,31 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' + +export const SNAPSHOT_ARCHIVE_REQUEST_MIGRATION = '2026-09-30-003 add snapshot archive requests' + +export async function addSnapshotArchiveRequestTable(knex: Knex): Promise { + if (!(await knex.schema.hasTable('snapshot_archive_requests'))) { + await knex.schema.createTable('snapshot_archive_requests', table => { + table.string('identityKey', 130).notNullable() + table.string('requestId', 64).notNullable() + table.string('claimToken', 64).notNullable() + table.string('state', 16).notNullable() + table.string('requestJson', 512).notNullable() + table.bigInteger('expiresAt').notNullable().index() + table.bigInteger('reservedBytes').notNullable() + table.string('archiveId', 64).nullable() + table.boolean('released').notNullable() + table.primary(['identityKey', 'requestId']) + }) + } +} + +export async function removeSnapshotArchiveRequestTable(knex: Knex): Promise { + if (await knex.schema.hasTable('snapshot_archive_requests')) { + const active = await knex('snapshot_archive_requests').where({ released: false }).first('requestId') + if (active !== undefined) { + throw new WERR_INVALID_OPERATION('Close snapshot archive requests before removing their schema') + } + } + await knex.schema.dropTableIfExists('snapshot_archive_requests') +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotCertificateIndexMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotCertificateIndexMigration.ts new file mode 100644 index 000000000..5a9359c8e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotCertificateIndexMigration.ts @@ -0,0 +1,615 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { runInSeries } from '../../utility/runInSeries' + +export const SNAPSHOT_CERTIFICATE_INDEX_MIGRATION = '2026-10-01-005 add snapshot certificate field key indexes' +const KEYS = 'snapshot_certificate_field_keys' +const PROGRESS = 'snapshot_certificate_index_progress' +const PAGE_ROWS = 256 +const keyColumns = ['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId'] +const indexes = [ + { + name: 'snapshot_certificate_parent', + columns: ['snapshotCertificateId', 'snapshotUserId', 'snapshotFieldName'] + }, + { + name: 'snapshot_certificate_lookup', + columns: ['snapshotFieldName', 'snapshotCertificateId', 'snapshotUserId'] + } +] +type Event = 'INSERT' | 'UPDATE' | 'DELETE' +interface Trigger { + name: string + table: string + timing: 'BEFORE' | 'AFTER' + event: Event + body: string + sql: string +} +interface TextDefinition { + charset: string | null + collation: string +} +function mysql(k: Knex): boolean { + return String(k.client.config.client).includes('mysql') +} +function normalized(sql: string): string { + return sql.replaceAll(/\s+/g, ' ').trim() +} +const textColumn = (name: string): boolean => name === 'snapshotFieldName' || name === 'afterFieldName' + +async function sourceText(k: Knex): Promise { + if (mysql(k)) { + const [tables]: Array> = await k.raw( + 'SELECT TABLE_NAME AS name, ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?, ?) ORDER BY TABLE_NAME', + ['certificate_fields', 'certificates'] + ) + if (tables.length !== 2 || tables.some(table => table.engine !== 'InnoDB')) + throw new WERR_INVALID_OPERATION('Snapshot certificate source requires transactional tables') + const [columns]: Array< + Array<{ + type: string + nullable: string + charset: string + collation: string + }> + > = await k.raw( + 'SELECT COLUMN_TYPE AS type, IS_NULLABLE AS nullable, CHARACTER_SET_NAME AS charset, COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?', + ['certificate_fields', 'fieldName'] + ) + const column = columns[0] + if ( + columns.length !== 1 || + column?.type !== 'varchar(100)' || + column.nullable !== 'NO' || + !/^\w+$/.test(column.charset) || + !/^\w+$/.test(column.collation) + ) + throw new WERR_INVALID_OPERATION('Unsupported snapshot certificate field definition') + return { charset: column.charset, collation: column.collation } + } + const unique: Array<{ name: string; unique: number; partial: number }> = await k.raw('PRAGMA index_list(??)', [ + 'certificate_fields' + ]) + for (const index of unique) { + if (index.unique !== 1 || index.partial !== 0) continue + const info: Array<{ + name: string + key: number + desc: number + coll: string + }> = await k.raw('PRAGMA index_xinfo(??)', [index.name]) + const parts = info.filter(part => part.key === 1) + if ( + parts.length !== 2 || + parts[0]?.name !== 'fieldName' || + parts[1]?.name !== 'certificateId' || + parts.some(part => part.desc !== 0) || + parts[1].coll !== 'BINARY' || + !['BINARY', 'NOCASE', 'RTRIM'].includes(parts[0].coll) + ) + continue + // A forced unique index must satisfy the declared column order itself. + // A separately collated index that needs a sort cannot define our cursor. + const plan: Array<{ detail: string }> = await k.raw( + 'EXPLAIN QUERY PLAN SELECT fieldName, certificateId FROM certificate_fields INDEXED BY ?? ORDER BY fieldName, certificateId LIMIT 1', + [index.name] + ) + if (plan.some(step => step.detail.includes('TEMP B-TREE'))) continue + return { charset: null, collation: parts[0].coll } + } + throw new WERR_INVALID_OPERATION('Unsupported snapshot certificate field order') +} + +function fieldType(text: TextDefinition): string { + const charset = text.charset === null ? '' : ' CHARACTER SET ' + text.charset + return `varchar(100)${charset} COLLATE ${text.collation}` +} +function insertMembership(isMysql: boolean, select: string, bit: number): string { + const merge = isMysql + ? ` ON DUPLICATE KEY UPDATE snapshotMembership = ${KEYS}.snapshotMembership | ${bit}` + : ` ON CONFLICT(${keyColumns.join(', ')}) DO UPDATE SET snapshotMembership = snapshotMembership | ${bit}` + return `INSERT INTO ${KEYS} (${keyColumns.join(', ')}, snapshotMembership) ${select}${merge};` +} +function trigger( + isMysql: boolean, + name: string, + table: string, + timing: Trigger['timing'], + event: Event, + statements: string, + changed?: string +): Trigger { + const body = + isMysql && changed !== undefined ? `BEGIN IF ${changed} THEN ${statements} END IF; END` : `BEGIN ${statements} END` + let qualifier = '' + if (isMysql) qualifier = ' FOR EACH ROW' + else if (changed !== undefined) qualifier = ` WHEN ${changed}` + return { + name, + table, + timing, + event, + body, + sql: `CREATE TRIGGER ${name} ${timing} ${event} ON ${table}${qualifier} ${body}` + } +} + +export function snapshotCertificateIndexTriggers(isMysql: boolean): { + observers: Trigger[] + producers: Trigger[] +} { + const different = (key: string): string => + isMysql ? `NOT (OLD.${key} <=> NEW.${key})` : `OLD.${key} IS NOT NEW.${key}` + const fieldChanged = isMysql + ? 'NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY))' + : 'CAST(OLD.fieldName AS BLOB) IS NOT CAST(NEW.fieldName AS BLOB)' + const changed = `${different('userId')} OR ${fieldChanged} OR ${different('certificateId')}` + const ownerChanged = `${different('userId')} OR ${different('certificateId')}` + const lock = isMysql ? ' FOR SHARE' : '' + const remove = `DELETE FROM ${KEYS} WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId;` + const add = + insertMembership(isMysql, 'VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1)', 1) + + ' ' + + insertMembership( + isMysql, + `SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId${lock}`, + 2 + ) + const where = 'snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId' + const subtract = `UPDATE ${KEYS} SET snapshotMembership = snapshotMembership & 1 WHERE ${where}; DELETE FROM ${KEYS} WHERE ${where} AND snapshotMembership = 0;` + // Direct memberships exist even when parent ownership is absent or differs. + // They supply a bounded parent prefix without changing legacy source indexes. + const join = isMysql ? 'JOIN' : 'CROSS JOIN' + const append = insertMembership( + isMysql, + `SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM ${KEYS} k ${join} certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId${lock}`, + 2 + ) + const prefix = 'snapshot_certificate' + return { + observers: [ + trigger(isMysql, `${prefix}_field_delete`, 'certificate_fields', 'AFTER', 'DELETE', remove), + trigger(isMysql, `${prefix}_field_before_update`, 'certificate_fields', 'BEFORE', 'UPDATE', remove, changed), + trigger(isMysql, `${prefix}_parent_delete`, 'certificates', 'AFTER', 'DELETE', subtract), + trigger(isMysql, `${prefix}_parent_before_update`, 'certificates', 'BEFORE', 'UPDATE', subtract, ownerChanged) + ], + producers: [ + trigger(isMysql, `${prefix}_field_insert`, 'certificate_fields', 'AFTER', 'INSERT', add), + trigger(isMysql, `${prefix}_field_after_update`, 'certificate_fields', 'AFTER', 'UPDATE', add, changed), + trigger(isMysql, `${prefix}_parent_insert`, 'certificates', 'AFTER', 'INSERT', append), + trigger(isMysql, `${prefix}_parent_after_update`, 'certificates', 'AFTER', 'UPDATE', append, ownerChanged) + ] + } +} +async function validateTrigger(k: Knex, expected: Trigger): Promise { + if (!mysql(k)) { + const row: { sql: string } | undefined = await k('sqlite_master') + .where({ type: 'trigger', name: expected.name }) + .first('sql') + if (row === undefined) return false + if (normalized(row.sql) !== normalized(expected.sql)) + throw new WERR_INVALID_OPERATION('Snapshot certificate trigger definition mismatch') + return true + } + const [rows]: Array> = await k.raw( + 'SELECT EVENT_MANIPULATION AS event, ACTION_TIMING AS timing, EVENT_OBJECT_TABLE AS tableName, ACTION_STATEMENT AS body FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = DATABASE() AND TRIGGER_NAME = ?', + [expected.name] + ) + if (!Array.isArray(rows)) throw new WERR_INVALID_OPERATION('Invalid snapshot certificate trigger metadata') + if (rows.length === 0) return false + const row = rows[0] + if ( + rows.length !== 1 || + row?.event !== expected.event || + row.timing !== expected.timing || + row.tableName !== expected.table || + normalized(row.body) !== normalized(expected.body) + ) { + throw new WERR_INVALID_OPERATION('Snapshot certificate trigger definition mismatch') + } + return true +} + +interface Column { + name: string + type: 'integer' | 'boolean' | 'varchar(100)' + unsigned?: boolean + primary: number +} +function tableColumns(keys: boolean): Column[] { + if (keys) + return [ + { name: 'snapshotUserId', type: 'integer', unsigned: true, primary: 1 }, + { name: 'snapshotFieldName', type: 'varchar(100)', primary: 2 }, + { + name: 'snapshotCertificateId', + type: 'integer', + unsigned: true, + primary: 3 + }, + { + name: 'snapshotMembership', + type: 'integer', + unsigned: true, + primary: 0 + } + ] + return [ + { name: 'snapshotTableId', type: 'integer', primary: 1 }, + { name: 'started', type: 'boolean', primary: 0 }, + { name: 'afterFieldName', type: 'varchar(100)', primary: 0 }, + { name: 'afterCertificateId', type: 'integer', unsigned: true, primary: 0 }, + { name: 'complete', type: 'boolean', primary: 0 } + ] +} +async function sqliteIndex(k: Knex, name: string, columns: string[], text: TextDefinition): Promise { + const rows: Array<{ name: string; desc: number; coll: string; key: number }> = await k.raw('PRAGMA index_xinfo(??)', [ + name + ]) + const parts = rows.filter(row => row.key === 1) + return ( + parts.length === columns.length && + parts.every( + (row, i) => + row.name === columns[i] && row.desc === 0 && row.coll === (textColumn(row.name) ? text.collation : 'BINARY') + ) + ) +} + +async function sqliteTable( + k: Knex, + table: string, + keys: boolean, + secondary: boolean, + text: TextDefinition +): Promise { + const expected = tableColumns(keys) + const columns: Array<{ + name: string + type: string + notnull: number + dflt_value: unknown + pk: number + hidden: number + }> = await k.raw('PRAGMA table_xinfo(??)', [table]) + if ( + !Array.isArray(columns) || + columns.length !== expected.length || + columns.some((column, i) => { + const field = expected[i] + return ( + column.name !== field.name || + column.type.toLowerCase() !== field.type || + column.notnull !== (!keys && i === 0 ? 0 : 1) || + column.dflt_value !== null || + column.pk !== field.primary || + column.hidden !== 0 + ) + }) + ) + return false + const existing: Array<{ + name: string + unique: number + origin: string + partial: number + }> = await k.raw('PRAGMA index_list(??)', [table]) + if (existing.some(index => index.unique !== 0 && (index.origin !== 'pk' || index.partial !== 0))) return false + if (!keys) return true + const primary = existing.find(index => index.origin === 'pk') + if (primary === undefined || !(await sqliteIndex(k, primary.name, keyColumns, text))) return false + if (!secondary) return true + for (const expectedIndex of indexes) { + const found = existing.find(index => index.name === expectedIndex.name) + if (found?.unique !== 0 || found.partial !== 0 || !(await sqliteIndex(k, found.name, expectedIndex.columns, text))) + return false + } + return true +} + +async function mysqlTable( + k: Knex, + table: string, + keys: boolean, + secondary: boolean, + text: TextDefinition +): Promise { + const [tables]: Array> = await k.raw( + 'SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?', + [table] + ) + if (tables.length !== 1 || tables[0]?.engine !== 'InnoDB') return false + const expected = tableColumns(keys) + const [columns]: Array< + Array<{ + name: string + type: string + nullable: string + defaultValue: unknown + extra: string + charset: string | null + collation: string | null + }> + > = await k.raw( + 'SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra, CHARACTER_SET_NAME AS charset, COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION', + [table] + ) + if ( + !Array.isArray(columns) || + columns.length !== expected.length || + columns.some((column, i) => { + const field = expected[i] + const integerType = field.unsigned === true ? 'int unsigned' : 'int' + const scalarType = field.type === 'boolean' ? 'tinyint' : integerType + const type = field.type === 'varchar(100)' ? field.type : scalarType + return ( + column.name !== field.name || + (textColumn(column.name) ? column.type : column.type.replaceAll(/\(\d+\)/g, '')) !== type || + column.charset !== (textColumn(column.name) ? text.charset : null) || + column.collation !== (textColumn(column.name) ? text.collation : null) || + column.nullable !== 'NO' || + column.defaultValue !== null || + column.extra !== '' + ) + }) + ) + return false + const [parts]: Array< + Array<{ + name: string + columnName: string + nonUnique: number + direction: string + prefix: unknown + }> + > = await k.raw( + 'SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX', + [table] + ) + if (!Array.isArray(parts) || parts.some(index => index.name !== 'PRIMARY' && Number(index.nonUnique) !== 1)) + return false + const required = [ + { name: 'PRIMARY', columns: keys ? keyColumns : ['snapshotTableId'] }, + ...(keys && secondary ? indexes : []) + ] + return required.every(index => { + const found = parts.filter(part => part.name === index.name) + return ( + found.length === index.columns.length && + found.every((part, i) => part.columnName === index.columns[i] && part.direction === 'A' && part.prefix === null) + ) + }) +} + +async function validateTable(k: Knex, table: string, text: TextDefinition, secondary = true): Promise { + const valid = mysql(k) + ? await mysqlTable(k, table, table === KEYS, secondary, text) + : await sqliteTable(k, table, table === KEYS, secondary, text) + if (!valid) throw new WERR_INVALID_OPERATION('Snapshot certificate table definition mismatch') +} + +async function ensureTables(k: Knex, text: TextDefinition): Promise { + if (!(await k.schema.hasTable(KEYS))) { + await k.schema.createTable(KEYS, t => { + if (mysql(k)) void t.engine('InnoDB') + t.integer('snapshotUserId').unsigned().notNullable() + t.specificType('snapshotFieldName', fieldType(text)).notNullable() + t.integer('snapshotCertificateId').unsigned().notNullable() + t.integer('snapshotMembership').unsigned().notNullable() + t.primary(keyColumns) + }) + } + // MySQL DDL commits independently. Resume an interrupted creation between + // the table and either auxiliary index without trusting a conflicting index. + await validateTable(k, KEYS, text, false) + await runInSeries(indexes, async index => { + const exists = mysql(k) + ? ( + await k.raw( + 'SELECT INDEX_NAME FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1', + [KEYS, index.name] + ) + )[0].length !== 0 + : (await k('sqlite_master').where({ type: 'index', name: index.name }).first('name')) !== undefined + if (!exists) + await k.schema.alterTable(KEYS, t => { + t.index(index.columns, index.name) + }) + }) + await validateTable(k, KEYS, text) + if (!(await k.schema.hasTable(PROGRESS))) { + await k.schema.createTable(PROGRESS, t => { + if (mysql(k)) void t.engine('InnoDB') + t.integer('snapshotTableId').primary() + t.boolean('started').notNullable() + t.specificType('afterFieldName', fieldType(text)).notNullable() + t.integer('afterCertificateId').unsigned().notNullable() + t.boolean('complete').notNullable() + }) + } + await validateTable(k, PROGRESS, text) +} + +interface Position { + started: boolean | number + afterFieldName: string + afterCertificateId: number + complete: boolean | number +} +function validPosition(state: Position | undefined): state is Position { + return ( + state !== undefined && + [false, true, 0, 1].includes(state.started) && + typeof state.afterFieldName === 'string' && + Array.from(state.afterFieldName).length <= 100 && + Number.isSafeInteger(state.afterCertificateId) && + state.afterCertificateId >= 0 && + [false, true, 0, 1].includes(state.complete) && + (state.started === true || state.started === 1 + ? state.afterCertificateId > 0 + : state.afterCertificateId === 0 && state.afterFieldName === '') + ) +} +function positive(value: number | undefined): number { + if (value === undefined || !Number.isSafeInteger(value) || value < 1) + throw new WERR_INVALID_OPERATION('Invalid snapshot certificate source key') + return value +} +function validateSourceRow(row: { userId: number; fieldName: string; certificateId: number }): void { + positive(row.userId) + positive(row.certificateId) + if (typeof row.fieldName !== 'string' || Array.from(row.fieldName).length > 100) + throw new WERR_INVALID_OPERATION('Invalid snapshot certificate source key') +} +async function bootstrapPage(k: Knex): Promise { + return await k.transaction(async trx => { + if (!mysql(k)) + await trx(PROGRESS) + .where('snapshotTableId', 0) + .update({ started: trx.ref('started') }) + const progress = trx(PROGRESS).where('snapshotTableId', 0) + if (mysql(k)) void progress.forUpdate() + const state: Position | undefined = await progress.first() + if (!validPosition(state)) throw new WERR_INVALID_OPERATION('Invalid snapshot certificate bootstrap position') + if (state.complete === true || state.complete === 1) return true + const source = trx('certificate_fields') + .select('userId', 'fieldName', 'certificateId') + .orderBy(['fieldName', 'certificateId']) + .limit(PAGE_ROWS) + if (mysql(k)) void source.forUpdate() + if (state.started === true || state.started === 1) { + if (mysql(k)) + void source.whereRaw('(fieldName > ? OR (fieldName = ? AND certificateId > ?))', [ + state.afterFieldName, + state.afterFieldName, + state.afterCertificateId + ]) + else void source.whereRaw('(fieldName, certificateId) > (?, ?)', [state.afterFieldName, state.afterCertificateId]) + } + const rows: Array<{ + userId: number + fieldName: string + certificateId: number + }> = await source + rows.forEach(validateSourceRow) + if (rows.length !== 0) { + const parentQuery = trx('certificates') + .select('certificateId', 'userId') + .whereIn( + 'certificateId', + [...new Set(rows.map(row => row.certificateId))].sort((a, b) => a - b) + ) + .orderBy('certificateId') + if (mysql(k)) void parentQuery.forShare() + const parents: Array<{ certificateId: number; userId: number }> = await parentQuery + const owners = new Map(parents.map(row => [positive(row.certificateId), positive(row.userId)])) + await runInSeries([1, 2], async bit => { + const memberships = rows.flatMap(row => { + const owner = bit === 1 ? row.userId : owners.get(row.certificateId) + return owner === undefined + ? [] + : [ + { + snapshotUserId: owner, + snapshotFieldName: row.fieldName, + snapshotCertificateId: row.certificateId, + snapshotMembership: bit + } + ] + }) + if (memberships.length !== 0) + await trx(KEYS) + .insert(memberships) + .onConflict(keyColumns) + .merge({ + snapshotMembership: trx.raw('?? | ?', ['snapshotMembership', bit]) + }) + }) + } + const last = rows.at(-1), + complete = rows.length < PAGE_ROWS + await trx(PROGRESS) + .where('snapshotTableId', 0) + .update({ + started: last === undefined ? state.started : true, + afterFieldName: last?.fieldName ?? state.afterFieldName, + afterCertificateId: last?.certificateId ?? state.afterCertificateId, + complete + }) + return complete + }) +} +export async function addSnapshotCertificateIndexes(k: Knex): Promise { + if (mysql(k) && k.isTransaction) + throw new WERR_INVALID_OPERATION( + 'Snapshot certificate migration requires independent DDL and bootstrap transactions' + ) + const text = await sourceText(k) + await ensureTables(k, text) + const { observers, producers } = snapshotCertificateIndexTriggers(mysql(k)) + await runInSeries([...observers, ...producers], async expected => { + if (!(await validateTrigger(k, expected))) await k.raw(expected.sql) + }) + await k(PROGRESS) + .insert({ + snapshotTableId: 0, + started: false, + afterFieldName: '', + afterCertificateId: 0, + complete: false + }) + .onConflict('snapshotTableId') + .ignore() + let complete = false + function* unfinishedPages() { + while (!complete) yield undefined + } + await runInSeries(unfinishedPages(), async () => { + complete = await bootstrapPage(k) + }) +} +/** Readers must be drained before removal; source rows and indexes stay intact. */ +export async function removeSnapshotCertificateIndexes(k: Knex): Promise { + if (mysql(k) && k.isTransaction) + throw new WERR_INVALID_OPERATION( + 'Snapshot certificate migration requires independent DDL and bootstrap transactions' + ) + const text = await sourceText(k) + if (await k.schema.hasTable(KEYS)) await validateTable(k, KEYS, text) + if (await k.schema.hasTable(PROGRESS)) await validateTable(k, PROGRESS, text) + const { observers, producers } = snapshotCertificateIndexTriggers(mysql(k)), + ordered = [...producers, ...observers] + await runInSeries(ordered, async expected => { + await validateTrigger(k, expected) + }) + await runInSeries(ordered, async expected => { + await k.raw('DROP TRIGGER IF EXISTS ??', [expected.name]) + }) + await k.schema.dropTableIfExists(PROGRESS) + await k.schema.dropTableIfExists(KEYS) +} +/** Resolve adoption inside the same retained view as the source header/pages. */ +export async function readSnapshotCertificateIndexState(k: Knex, config?: Knex.MigratorConfig): Promise { + const tableName = config?.tableName ?? 'knex_migrations', + schema = k.schema + if (config?.schemaName !== undefined) void schema.withSchema(config.schemaName) + if (!(await schema.hasTable(tableName))) return false + const journal = k(tableName).where('name', SNAPSHOT_CERTIFICATE_INDEX_MIGRATION) + if (config?.schemaName !== undefined) void journal.withSchema(config.schemaName) + if ((await journal.first('name')) === undefined) return false + if (!(await k.schema.hasTable(KEYS)) || !(await k.schema.hasTable(PROGRESS))) + throw new WERR_INVALID_OPERATION('Snapshot certificate index migration is incomplete') + const text = await sourceText(k) + await validateTable(k, KEYS, text) + await validateTable(k, PROGRESS, text) + const states: Array = await k(PROGRESS).select('*').limit(2) + if ( + states.length !== 1 || + states[0]?.snapshotTableId !== 0 || + !validPosition(states[0]) || + (states[0].complete !== true && states[0].complete !== 1) + ) + throw new WERR_INVALID_OPERATION('Snapshot certificate index migration is incomplete') + return true +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexBootstrap.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexBootstrap.ts new file mode 100644 index 000000000..effac4a8e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexBootstrap.ts @@ -0,0 +1,148 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../utility/runInSeries' +import { EDGES, GUARDS, PROGRESS, PAGE_ROWS, mysql, invalid } from './snapshotGlobalIndexModel' + +export interface Position { + afterRowId: number + complete: boolean | number +} +export function validPosition(state: Position | undefined): state is Position { + return ( + state !== undefined && + Number.isSafeInteger(state.afterRowId) && + state.afterRowId >= 0 && + [false, true, 0, 1].includes(state.complete) + ) +} +function positive(value: number): number { + if (!Number.isSafeInteger(value) || value < 1) invalid('Invalid snapshot global source key') + return value +} +interface SourceRow { + transactionId: number + userId: number + provenTxId: number | null + txid: string | null + txidBytes: number +} +interface RequestRow { + provenTxReqId: number + provenTxId: number | null +} +async function currentProof(k: Knex, proofId: number): Promise { + positive(proofId) + await k(GUARDS) + .insert({ proofId, present: false }) + .onConflict('proofId') + .merge({ proofId: k.ref('proofId') }) + // This lock is shared with proof insert/delete triggers, including absence. + // UPDATE reads the current source after acquiring the auxiliary target lock. + if (mysql(k)) + await k.raw( + 'UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=?', + [proofId] + ) + else + await k(GUARDS) + .where('proofId', proofId) + .update({ + present: k.raw('EXISTS(SELECT 1 FROM proven_txs WHERE provenTxId=?)', [proofId]) + }) +} +async function bootstrapRow(k: Knex, row: SourceRow): Promise { + positive(row.transactionId) + positive(row.userId) + if ( + !Number.isSafeInteger(row.txidBytes) || + row.txidBytes < 0 || + row.txidBytes > 256 || + (row.txid !== null && (typeof row.txid !== 'string' || Array.from(row.txid).length > 64)) + ) + invalid('Invalid snapshot global transaction key') + let request: RequestRow | undefined + if (row.txid !== null) { + const query = k('proven_tx_reqs').select('provenTxReqId', 'provenTxId').where('txid', row.txid) + if (mysql(k)) void query.forShare() + request = await query.first() + if (request !== undefined) positive(request.provenTxReqId) + } + const proofs = [ + ...new Set( + [row.provenTxId, request?.provenTxId].filter((value): value is number => value !== null && value !== undefined) + ) + ].sort((a, b) => a - b) + await runInSeries(proofs, async proofId => { + await currentProof(k, proofId) + }) + const values: Array<{ requestId: number; tableId: number; rowId: number }> = [] + if (row.provenTxId !== null) values.push({ requestId: 0, tableId: 1, rowId: positive(row.provenTxId) }) + if (request !== undefined) { + values.push({ + requestId: request.provenTxReqId, + tableId: 0, + rowId: request.provenTxReqId + }) + if (request.provenTxId !== null) + values.push({ + requestId: request.provenTxReqId, + tableId: 1, + rowId: positive(request.provenTxId) + }) + } + if (values.length !== 0) + await k(EDGES) + .insert( + values.map(value => ({ + ...value, + transactionId: row.transactionId, + userId: row.userId + })) + ) + .onConflict(['transactionId', 'requestId', 'tableId', 'rowId']) + .merge({ transactionId: k.ref('transactionId') }) +} +export async function bootstrapPage(k: Knex): Promise { + return await k.transaction(async trx => { + if (!mysql(k)) + await trx(PROGRESS) + .where('id', 0) + .update({ complete: trx.ref('complete') }) + const position = trx(PROGRESS).where('id', 0) + if (mysql(k)) void position.forUpdate() + const state: Position | undefined = await position.first() + if (!validPosition(state)) invalid('Invalid snapshot global bootstrap position') + if (state.complete === true || state.complete === 1) return true + if (state.afterRowId === 0) { + // The initial position is below every supported source key. Do not mark a + // malformed legacy SQLite store complete while silently skipping its rows. + const unsupported = trx('transactions').select('transactionId').where('transactionId', '<=', 0) + if (mysql(k)) void unsupported.forUpdate() + if ((await unsupported.first()) !== undefined) invalid('Invalid snapshot global source key') + } + const length = mysql(k) ? 'octet_length(txid)' : 'length(cast(txid AS blob))' + const source = trx('transactions') + .select( + 'transactionId', + 'userId', + 'provenTxId', + trx.raw(`CASE WHEN ${length} <= ? THEN txid END AS txid`, [256]), + trx.raw(`COALESCE(${length},?) AS txidBytes`, [0]) + ) + .where('transactionId', '>', state.afterRowId) + .orderBy('transactionId') + .limit(PAGE_ROWS) + if (mysql(k)) void source.forUpdate() + const rows: SourceRow[] = await source + await runInSeries(rows, async row => { + await bootstrapRow(trx, row) + }) + const complete = rows.length < PAGE_ROWS + await trx(PROGRESS) + .where('id', 0) + .update({ + afterRowId: rows.at(-1)?.transactionId ?? state.afterRowId, + complete + }) + return complete + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexMigration.ts new file mode 100644 index 000000000..95f4b6534 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexMigration.ts @@ -0,0 +1,130 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../utility/runInSeries' +import { snapshotGlobalIndexTriggers, type SnapshotGlobalIndexTrigger } from './snapshotGlobalIndexTriggers' +import { tables, PROGRESS, mysql, normalized, invalid, type Table } from './snapshotGlobalIndexModel' +import { mysqlParts, mysqlTable, validateMysqlSource } from './snapshotGlobalIndexMysql' +import { sqliteTable, validateSqliteSource } from './snapshotGlobalIndexSqlite' +import { validPosition, bootstrapPage, type Position } from './snapshotGlobalIndexBootstrap' + +export const SNAPSHOT_GLOBAL_INDEX_MIGRATION = '2026-10-01-006 add snapshot global reference indexes' +async function validateTable(k: Knex, table: Table, secondary = true): Promise { + if (!(mysql(k) ? await mysqlTable(k, table, secondary) : await sqliteTable(k, table, secondary))) + invalid('Snapshot global table definition mismatch') +} +async function ensureTable(k: Knex, table: Table): Promise { + if (!(await k.schema.hasTable(table.name))) + await k.schema.createTable(table.name, builder => { + if (mysql(k)) void builder.engine('InnoDB') + for (const column of table.columns) { + let field: Knex.ColumnBuilder + if (column.type === 'boolean') field = builder.boolean(column.name) + else if (column.type === 'biguint') field = builder.bigInteger(column.name).unsigned() + else if (column.type === 'uint') field = builder.integer(column.name).unsigned() + else field = builder.integer(column.name) + void field.notNullable() + } + void builder.primary(table.primary) + }) + await validateTable(k, table, false) + await runInSeries(table.indexes, async index => { + const exists = mysql(k) + ? (await mysqlParts(k, table.name)).some(part => part.name === index.name) + : (await k('sqlite_master').where({ type: 'index', name: index.name }).first('name')) !== undefined + if (!exists) + await k.schema.alterTable(table.name, builder => { + void builder.index(index.columns, index.name) + }) + }) + await validateTable(k, table) +} +async function validateSource(k: Knex): Promise { + if (mysql(k)) await validateMysqlSource(k) + else await validateSqliteSource(k) +} +async function validateTrigger(k: Knex, expected: SnapshotGlobalIndexTrigger): Promise { + if (!mysql(k)) { + const row: { sql: string } | undefined = await k('sqlite_master') + .where({ type: 'trigger', name: expected.name }) + .first('sql') + if (row === undefined) return false + if (normalized(row.sql) !== normalized(expected.sql)) invalid('Snapshot global trigger definition mismatch') + return true + } + const [rows]: Array> = await k.raw( + 'SELECT EVENT_MANIPULATION AS event,ACTION_TIMING AS timing,EVENT_OBJECT_TABLE AS tableName,ACTION_STATEMENT AS body FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?', + [expected.name] + ) + if (!Array.isArray(rows)) invalid('Invalid snapshot global trigger metadata') + if (rows.length === 0) return false + const row = rows[0] + if ( + rows.length !== 1 || + row?.event !== expected.event || + row.timing !== expected.timing || + row.tableName !== expected.table || + normalized(row.body) !== normalized(expected.body) + ) + invalid('Snapshot global trigger definition mismatch') + return true +} +export async function addSnapshotGlobalIndexes(k: Knex): Promise { + if (mysql(k) && k.isTransaction) + invalid('Snapshot global migration requires independent DDL and bootstrap transactions') + await validateSource(k) + await runInSeries(tables(), async table => { + await ensureTable(k, table) + }) + await runInSeries(snapshotGlobalIndexTriggers(mysql(k)), async trigger => { + if (!(await validateTrigger(k, trigger))) await k.raw(trigger.sql) + }) + await k(PROGRESS).insert({ id: 0, afterRowId: 0, complete: false }).onConflict('id').ignore() + let complete = false + function* unfinishedPages() { + while (!complete) yield undefined + } + await runInSeries(unfinishedPages(), async () => { + complete = await bootstrapPage(k) + }) +} +/** Drain retained readers before removal; all standard source rows/indexes remain. */ +export async function removeSnapshotGlobalIndexes(k: Knex): Promise { + if (mysql(k) && k.isTransaction) + invalid('Snapshot global migration requires independent DDL and bootstrap transactions') + await validateSource(k) + await runInSeries(tables(), async table => { + if (await k.schema.hasTable(table.name)) await validateTable(k, table) + }) + const triggers = snapshotGlobalIndexTriggers(mysql(k)).reverse() + await runInSeries(triggers, async trigger => { + await validateTrigger(k, trigger) + }) + await runInSeries(triggers, async trigger => { + await k.raw('DROP TRIGGER IF EXISTS ??', [trigger.name]) + }) + await runInSeries([...tables()].reverse(), async table => { + await k.schema.dropTableIfExists(table.name) + }) +} +export async function readSnapshotGlobalIndexState(k: Knex, config?: Knex.MigratorConfig): Promise { + const tableName = config?.tableName ?? 'knex_migrations', + schema = k.schema + if (config?.schemaName !== undefined) void schema.withSchema(config.schemaName) + if (!(await schema.hasTable(tableName))) return false + const journal = k(tableName).where('name', SNAPSHOT_GLOBAL_INDEX_MIGRATION) + if (config?.schemaName !== undefined) void journal.withSchema(config.schemaName) + if ((await journal.first('name')) === undefined) return false + await validateSource(k) + await runInSeries(tables(), async table => { + if (!(await k.schema.hasTable(table.name))) invalid('Snapshot global index migration is incomplete') + await validateTable(k, table) + }) + const states: Array = await k(PROGRESS).select('*').limit(2) + if ( + states.length !== 1 || + states[0]?.id !== 0 || + !validPosition(states[0]) || + (states[0].complete !== true && states[0].complete !== 1) + ) + invalid('Snapshot global index migration is incomplete') + return true +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexModel.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexModel.ts new file mode 100644 index 000000000..5b50755c4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexModel.ts @@ -0,0 +1,112 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +export const EDGES = 'snapshot_global_edges' +export const KEYS = 'snapshot_global_keys' +export const GUARDS = 'snapshot_global_guards' +export const PROGRESS = 'snapshot_global_index_progress' +export const PAGE_ROWS = 256 + +type ColumnType = 'int' | 'uint' | 'biguint' | 'boolean' +interface Column { + name: string + type: ColumnType +} +export interface Index { + name: string + columns: string[] +} +export interface Table { + name: string + columns: Column[] + primary: string[] + indexes: Index[] +} +const columns = (names: string[], type: ColumnType): Column[] => names.map(name => ({ name, type })) +// Build descriptors when the migration runs, so construction failures belong to +// the awaited migration operation rather than module initialization. +export function tables(): Table[] { + return [ + { + name: GUARDS, + columns: [...columns(['proofId'], 'uint'), ...columns(['present'], 'boolean')], + primary: ['proofId'], + indexes: [] + }, + { + name: KEYS, + columns: [ + ...columns(['tableId'], 'int'), + ...columns(['userId', 'rowId'], 'uint'), + ...columns(['refs'], 'biguint'), + ...columns(['present'], 'boolean') + ], + primary: ['tableId', 'userId', 'rowId'], + indexes: [ + { + name: 'snapshot_global_page', + columns: ['tableId', 'userId', 'present', 'rowId'] + }, + { + name: 'snapshot_global_target', + columns: ['tableId', 'rowId', 'userId'] + } + ] + }, + { + name: EDGES, + columns: [ + ...columns(['transactionId', 'requestId'], 'uint'), + ...columns(['tableId'], 'int'), + ...columns(['rowId', 'userId'], 'uint') + ], + primary: ['transactionId', 'requestId', 'tableId', 'rowId'], + indexes: [ + { + name: 'snapshot_global_request', + columns: ['requestId', 'transactionId'] + } + ] + }, + { + name: PROGRESS, + columns: [...columns(['id'], 'int'), ...columns(['afterRowId'], 'uint'), ...columns(['complete'], 'boolean')], + primary: ['id'], + indexes: [] + } + ] +} + +export const mysql = (k: Knex): boolean => String(k.client.config.client).includes('mysql') +export const normalized = (sql: string): string => sql.replaceAll(/\s+/g, ' ').trim() +export function invalid(message: string): never { + throw new WERR_INVALID_OPERATION(message) +} + +export const sources = [ + { + name: 'proven_txs', + key: 'provenTxId', + fields: [{ name: 'provenTxId', nullable: false, text: false }] + }, + { + name: 'proven_tx_reqs', + key: 'provenTxReqId', + fields: [ + { name: 'provenTxReqId', nullable: false, text: false }, + { name: 'provenTxId', nullable: true, text: false }, + { name: 'txid', nullable: false, text: true } + ] + }, + { + name: 'transactions', + key: 'transactionId', + fields: [ + { name: 'transactionId', nullable: false, text: false }, + { name: 'userId', nullable: false, text: false }, + { name: 'provenTxId', nullable: true, text: false }, + { name: 'txid', nullable: true, text: true } + ] + } +] +export type SourceDefinition = (typeof sources)[number] +export type SourceField = SourceDefinition['fields'][number] diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexMysql.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexMysql.ts new file mode 100644 index 000000000..bb90409f4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexMysql.ts @@ -0,0 +1,144 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../utility/runInSeries' +import { invalid, sources, type Table, type SourceDefinition, type SourceField } from './snapshotGlobalIndexModel' +interface MysqlPart { + name: string + columnName: string + nonUnique: number + direction: string + prefix: unknown +} +export async function mysqlParts(k: Knex, table: string): Promise { + const [parts]: MysqlPart[][] = await k.raw( + 'SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX', + [table] + ) + if (!Array.isArray(parts)) invalid('Invalid snapshot global index metadata') + return parts +} +function matchesMysqlIndex(parts: MysqlPart[], name: string, expected: string[]): boolean { + const found = parts.filter(part => part.name === name) + return ( + found.length === expected.length && + found.every((part, i) => part.columnName === expected[i] && part.direction === 'A' && part.prefix === null) + ) +} +export async function mysqlTable(k: Knex, table: Table, secondary: boolean): Promise { + const [engines]: Array> = await k.raw( + 'SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?', + [table.name] + ) + if (engines.length !== 1 || engines[0]?.engine !== 'InnoDB') return false + const [actual]: Array< + Array<{ + name: string + type: string + nullable: string + defaultValue: unknown + extra: string + }> + > = await k.raw( + 'SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,COLUMN_DEFAULT AS defaultValue,EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION', + [table.name] + ) + const types = { + int: 'int', + uint: 'int unsigned', + biguint: 'bigint unsigned', + boolean: 'tinyint' + } + if ( + !Array.isArray(actual) || + actual.length !== table.columns.length || + actual.some( + (column, i) => + column.name !== table.columns[i].name || + column.type.replaceAll(/\(\d+\)/g, '') !== types[table.columns[i].type] || + column.nullable !== 'NO' || + column.defaultValue !== null || + column.extra !== '' + ) + ) + return false + const parts = await mysqlParts(k, table.name) + if (parts.some(part => part.name !== 'PRIMARY' && Number(part.nonUnique) !== 1)) return false + if (!matchesMysqlIndex(parts, 'PRIMARY', table.primary)) return false + return !secondary || table.indexes.every(index => matchesMysqlIndex(parts, index.name, index.columns)) +} +interface MysqlSourceColumn { + name: string + type: string + nullable: string + extra: string + charset: string | null + collation: string | null +} +interface SourceText { + charset: string + collation: string +} +function mysqlSourceColumn(column: MysqlSourceColumn | undefined, field: SourceField, key: string): MysqlSourceColumn { + const type = field.text ? 'varchar(64)' : 'int unsigned' + if ( + column === undefined || + (field.text ? column.type : column.type.replaceAll(/\(\d+\)/g, '')) !== type || + column.nullable !== (field.nullable ? 'YES' : 'NO') || + (column.extra !== '' && !(field.name === key && column.extra === 'auto_increment')) + ) + invalid('Unsupported snapshot global source column') + return column +} +function mysqlSourceText(previous: SourceText | undefined, column: MysqlSourceColumn): SourceText { + if (column.charset === null || column.collation === null) invalid('Unsupported snapshot global source text') + if (previous !== undefined && (previous.charset !== column.charset || previous.collation !== column.collation)) + invalid('Snapshot global source comparisons require matching text definitions') + return { charset: column.charset, collation: column.collation } +} +function validateMysqlSourceIndexes(parts: MysqlPart[], source: SourceDefinition): void { + if (!matchesMysqlIndex(parts, 'PRIMARY', [source.key])) invalid('Unsupported snapshot global source key') + if (source.name === 'proven_txs') return + const candidates = [...new Set(parts.map(part => part.name))] + const indexed = candidates.some(name => { + const index = parts.filter(part => part.name === name) + return ( + index[0]?.columnName === 'txid' && + index[0].direction === 'A' && + index[0].prefix === null && + (source.name === 'transactions' || (index.length === 1 && Number(index[0].nonUnique) === 0)) + ) + }) + if (!indexed) invalid('Snapshot global source requires complete transaction lookup indexes') +} +export async function validateMysqlSource(k: Knex): Promise { + let text: SourceText | undefined + await runInSeries(sources, async source => { + const [engines]: Array> = await k.raw( + 'SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?', + [source.name] + ) + if (engines.length !== 1 || engines[0]?.engine !== 'InnoDB') + invalid('Snapshot global source requires transactional tables') + const [rules]: Array> = await k.raw( + 'SELECT UPDATE_RULE AS updateRule,DELETE_RULE AS deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME=?', + [source.name] + ) + // InnoDB cascades do not run the affected child's row triggers. Keep the + // standard RESTRICT/NO ACTION definitions; do not adopt a stale edge index. + const explicit = (rule: string): boolean => rule === 'RESTRICT' || rule === 'NO ACTION' + if (!Array.isArray(rules) || rules.some(rule => !explicit(rule.updateRule) || !explicit(rule.deleteRule))) + invalid('Snapshot global source requires explicit row mutations') + const [actual]: MysqlSourceColumn[][] = await k.raw( + 'SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,EXTRA AS extra,CHARACTER_SET_NAME AS charset,COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?', + [source.name] + ) + for (const field of source.fields) { + const column = mysqlSourceColumn( + actual.find(value => value.name === field.name), + field, + source.key + ) + if (field.text) text = mysqlSourceText(text, column) + } + validateMysqlSourceIndexes(await mysqlParts(k, source.name), source) + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexSqlite.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexSqlite.ts new file mode 100644 index 000000000..20c63a522 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexSqlite.ts @@ -0,0 +1,119 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../utility/runInSeries' +import { invalid, sources, type Table, type Index } from './snapshotGlobalIndexModel' +interface SqlitePart { + name: string + desc: number + coll: string + key: number +} +async function sqliteParts(k: Knex, name: string): Promise { + const rows: SqlitePart[] = await k.raw('PRAGMA index_xinfo(??)', [name]) + return rows.filter(row => row.key === 1) +} +export async function sqliteTable(k: Knex, table: Table, secondary: boolean): Promise { + const actual: Array<{ + name: string + type: string + notnull: number + dflt_value: unknown + pk: number + hidden: number + }> = await k.raw('PRAGMA table_xinfo(??)', [table.name]) + const types = { + int: 'integer', + uint: 'integer', + biguint: 'bigint', + boolean: 'boolean' + } + if ( + !Array.isArray(actual) || + actual.length !== table.columns.length || + actual.some( + (column, i) => + column.name !== table.columns[i].name || + column.type.toLowerCase() !== types[table.columns[i].type] || + column.notnull !== 1 || + column.dflt_value !== null || + column.pk !== table.primary.indexOf(column.name) + 1 || + column.hidden !== 0 + ) + ) + return false + const indexes: Array<{ + name: string + unique: number + origin: string + partial: number + }> = await k.raw('PRAGMA index_list(??)', [table.name]) + if (indexes.some(index => index.unique !== 0 && (index.origin !== 'pk' || index.partial !== 0))) return false + const required: Index[] = secondary ? [...table.indexes] : [] + if (table.primary.length > 1) { + const primary = indexes.find(index => index.origin === 'pk') + if (primary === undefined) return false + required.push({ name: primary.name, columns: table.primary }) + } + for (const index of required) { + const found = indexes.find(value => value.name === index.name) + if (found?.partial !== 0) return false + const parts = await sqliteParts(k, found.name) + if ( + parts.length !== index.columns.length || + parts.some((part, i) => part.name !== index.columns[i] || part.desc !== 0 || part.coll !== 'BINARY') + ) + return false + } + return true +} +async function sqliteTextOrder(k: Knex, table: string): Promise { + const indexes: Array<{ name: string; unique: number; partial: number }> = await k.raw('PRAGMA index_list(??)', [ + table + ]) + for (const index of indexes) { + if (index.partial !== 0 || (table === 'proven_tx_reqs' && index.unique !== 1)) continue + const parts = await sqliteParts(k, index.name) + if ( + parts[0]?.name !== 'txid' || + parts[0].desc !== 0 || + !['BINARY', 'NOCASE', 'RTRIM'].includes(parts[0].coll) || + (table === 'proven_tx_reqs' && parts.length !== 1) + ) + continue + const plan: Array<{ detail: string }> = await k.raw( + 'EXPLAIN QUERY PLAN SELECT txid FROM ?? INDEXED BY ?? ORDER BY txid LIMIT 1', + [table, index.name] + ) + if (!plan.some(step => step.detail.includes('TEMP B-TREE'))) return parts[0].coll + } + return invalid('Snapshot global source requires complete transaction lookup indexes') +} +export async function validateSqliteSource(k: Knex): Promise { + let collation: string | undefined + await runInSeries(sources, async source => { + const actual: Array<{ + name: string + type: string + notnull: number + pk: number + hidden: number + }> = await k.raw('PRAGMA table_xinfo(??)', [source.name]) + const primary = actual.filter(column => column.pk !== 0) + if (primary.length !== 1 || primary[0]?.name !== source.key || primary[0].pk !== 1) + invalid('Unsupported snapshot global source key') + for (const field of source.fields) { + const column = actual.find(value => value.name === field.name) + if ( + column?.type.toLowerCase() !== (field.text ? 'varchar(64)' : 'integer') || + column.hidden !== 0 || + (field.name !== source.key && column.notnull !== (field.nullable ? 0 : 1)) + ) + invalid('Unsupported snapshot global source column') + } + if (source.name !== 'proven_txs') { + const order = await sqliteTextOrder(k, source.name) + if (collation !== undefined && collation !== order) + invalid('Snapshot global source comparisons require matching text definitions') + collation = order + } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexTriggers.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexTriggers.ts new file mode 100644 index 000000000..17e9818ba --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotGlobalIndexTriggers.ts @@ -0,0 +1,121 @@ +/** Migration-owned trigger definitions; source indexes and rows are preserved. */ +export interface SnapshotGlobalIndexTrigger { + name: string + table: string + timing: 'BEFORE' | 'AFTER' + event: 'INSERT' | 'UPDATE' | 'DELETE' + body: string + sql: string +} +const edges = 'snapshot_global_edges', + keys = 'snapshot_global_keys', + guards = 'snapshot_global_guards' +export function snapshotGlobalIndexTriggers(isMysql: boolean): SnapshotGlobalIndexTrigger[] { + const statements: SnapshotGlobalIndexTrigger[] = [] + const trigger = ( + name: string, + table: string, + timing: 'BEFORE' | 'AFTER', + event: 'INSERT' | 'UPDATE' | 'DELETE', + body: string, + when?: string + ): void => { + const condition = isMysql && when !== undefined ? `IF ${when} THEN ` : '' + const end = when !== undefined && isMysql ? ' END IF;' : '' + const fullName = `snapshot_global_${name}` + const fullBody = `BEGIN ${condition}${body}${end} END` + let qualifier = '' + if (isMysql) qualifier = ' FOR EACH ROW' + else if (when !== undefined) qualifier = ' WHEN ' + when + const sql = `CREATE TRIGGER ${fullName} ${timing} ${event} ON ${table}${qualifier} ${fullBody}` + statements.push({ + name: fullName, + table, + timing, + event, + body: fullBody, + sql + }) + } + const edgeInsert = (selection: string): string => + `INSERT INTO ${edges} (transactionId,requestId,tableId,rowId,userId) ${selection}${isMysql ? ' ON DUPLICATE KEY UPDATE transactionId = ' + edges + '.transactionId' : ' ON CONFLICT(transactionId,requestId,tableId,rowId) DO NOTHING'};` + const ensureProof = (expression: string): string => + isMysql + ? `INSERT INTO ${guards} (proofId,present) VALUES (${expression},0) ON DUPLICATE KEY UPDATE proofId=${guards}.proofId; UPDATE ${guards} g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=${expression};` + : `INSERT INTO ${guards} (proofId,present) SELECT ${expression},0 WHERE ${expression} IS NOT NULL ON CONFLICT(proofId) DO NOTHING; UPDATE ${guards} SET present=EXISTS(SELECT 1 FROM proven_txs WHERE provenTxId=${expression}) WHERE proofId=${expression};` + const guardPresence = `CASE WHEN NEW.tableId=0 THEN 1 ELSE (SELECT present FROM ${guards} WHERE proofId=NEW.rowId${isMysql ? ' FOR SHARE' : ''}) END` + const increase = `INSERT INTO ${keys} (tableId,userId,rowId,refs,present) VALUES (NEW.tableId,NEW.userId,NEW.rowId,1,${guardPresence})${isMysql ? ' ON DUPLICATE KEY UPDATE refs=' + keys + '.refs+1' : ' ON CONFLICT(tableId,userId,rowId) DO UPDATE SET refs=refs+1'};` + const countWhere = 'tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId' + const collect = `DELETE FROM ${guards} WHERE proofId=OLD.rowId AND OLD.tableId=1 AND NOT EXISTS(SELECT 1 FROM ${keys} WHERE tableId=1 AND rowId=OLD.rowId);` + trigger( + 'edge_delete', + edges, + 'AFTER', + 'DELETE', + `UPDATE ${keys} SET refs=refs-1 WHERE ${countWhere}; DELETE FROM ${keys} WHERE ${countWhere} AND refs=0; ${collect}` + ) + trigger('edge_insert', edges, 'AFTER', 'INSERT', increase) + const diff = (field: string): string => + isMysql ? `NOT (OLD.${field} <=> NEW.${field})` : `OLD.${field} IS NOT NEW.${field}` + const txnChanged = ['transactionId', 'userId', 'txid', 'provenTxId'].map(diff).join(' OR ') + const reqChanged = ['provenTxReqId', 'txid', 'provenTxId'].map(diff).join(' OR ') + trigger('tx_delete', 'transactions', 'AFTER', 'DELETE', `DELETE FROM ${edges} WHERE transactionId=OLD.transactionId;`) + trigger( + 'tx_before_update', + 'transactions', + 'BEFORE', + 'UPDATE', + `DELETE FROM ${edges} WHERE transactionId=OLD.transactionId;`, + txnChanged + ) + trigger('req_delete', 'proven_tx_reqs', 'AFTER', 'DELETE', `DELETE FROM ${edges} WHERE requestId=OLD.provenTxReqId;`) + trigger( + 'req_before_update', + 'proven_tx_reqs', + 'BEFORE', + 'UPDATE', + `DELETE FROM ${edges} WHERE requestId=OLD.provenTxReqId;`, + reqChanged + ) + const proofPresence = (prefix: 'OLD' | 'NEW', present: 0 | 1): string => { + const merge = isMysql + ? ` ON DUPLICATE KEY UPDATE present=${present}` + : ` ON CONFLICT(proofId) DO UPDATE SET present=${present}` + return `INSERT INTO ${guards} (proofId,present) VALUES (${prefix}.provenTxId,${present})${merge}; UPDATE ${keys} SET present=${present} WHERE tableId=1 AND rowId=${prefix}.provenTxId; DELETE FROM ${guards} WHERE proofId=${prefix}.provenTxId AND NOT EXISTS(SELECT 1 FROM ${keys} WHERE tableId=1 AND rowId=${prefix}.provenTxId);` + } + trigger('proof_delete', 'proven_txs', 'AFTER', 'DELETE', proofPresence('OLD', 0)) + trigger('proof_before_update', 'proven_txs', 'BEFORE', 'UPDATE', proofPresence('OLD', 0), diff('provenTxId')) + trigger('proof_insert', 'proven_txs', 'AFTER', 'INSERT', proofPresence('NEW', 1)) + trigger('proof_after_update', 'proven_txs', 'AFTER', 'UPDATE', proofPresence('NEW', 1), diff('provenTxId')) + let txnBody: string, + declarations = '' + if (isMysql) { + declarations = + 'DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; ' + txnBody = `IF NEW.provenTxId IS NOT NULL THEN ${ensureProof('NEW.provenTxId')} ${edgeInsert('VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId)')} END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN ${edgeInsert('VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId)')} IF requestedProof IS NOT NULL THEN ${ensureProof('requestedProof')} ${edgeInsert('VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId)')} END IF; END IF;` + } else { + const reqProof = '(SELECT provenTxId FROM proven_tx_reqs WHERE txid=NEW.txid)' + txnBody = `${ensureProof('NEW.provenTxId')} ${ensureProof(reqProof)} ${edgeInsert('SELECT NEW.transactionId,0,1,NEW.provenTxId,NEW.userId WHERE NEW.provenTxId IS NOT NULL')} ${edgeInsert('SELECT NEW.transactionId,provenTxReqId,0,provenTxReqId,NEW.userId FROM proven_tx_reqs WHERE txid=NEW.txid')} ${edgeInsert('SELECT NEW.transactionId,provenTxReqId,1,provenTxId,NEW.userId FROM proven_tx_reqs WHERE txid=NEW.txid AND provenTxId IS NOT NULL')}` + } + // DECLARE belongs before the optional IF in a MySQL trigger body. + trigger('tx_insert', 'transactions', 'AFTER', 'INSERT', declarations + txnBody) + if (isMysql) + trigger( + 'tx_after_update', + 'transactions', + 'AFTER', + 'UPDATE', + `${declarations} IF ${txnChanged} THEN ${txnBody} END IF;` + ) + else trigger('tx_after_update', 'transactions', 'AFTER', 'UPDATE', txnBody, txnChanged) + const current = isMysql ? ' FOR SHARE' : '' + const ensureReqProof = isMysql + ? `IF NEW.provenTxId IS NOT NULL THEN ${ensureProof('NEW.provenTxId')} END IF;` + : ensureProof('NEW.provenTxId') + const requestEdges = `SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId${current}` + const proofEdges = `SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId${current}` + const reqBody = `${ensureReqProof} ${edgeInsert(requestEdges)} ${edgeInsert(proofEdges)} DELETE FROM ${guards} WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM ${keys} WHERE tableId=1 AND rowId=NEW.provenTxId);` + trigger('req_insert', 'proven_tx_reqs', 'AFTER', 'INSERT', reqBody) + trigger('req_after_update', 'proven_tx_reqs', 'AFTER', 'UPDATE', reqBody, reqChanged) + return statements +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotProfileIndexMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotProfileIndexMigration.ts new file mode 100644 index 000000000..1e96cdc1e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotProfileIndexMigration.ts @@ -0,0 +1,333 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { runInSeries } from '../../utility/runInSeries' + +export const SNAPSHOT_PROFILE_INDEX_MIGRATION = '2026-10-01-003 add snapshot profile key indexes' + +export const snapshotProfileTables = [ + { table: 'transactions', key: 'transactionId' }, + { table: 'outputs', key: 'outputId' }, + { table: 'certificates', key: 'certificateId' }, + { table: 'tx_labels', key: 'txLabelId' }, + { table: 'output_baskets', key: 'basketId' }, + { table: 'output_tags', key: 'outputTagId' }, + { table: 'commissions', key: 'commissionId' }, + { table: 'sync_states', key: 'syncStateId' } +] as const + +const KEYS = 'snapshot_profile_keys' +const PROGRESS = 'snapshot_profile_index_progress' +const PAGE_ROWS = 256 + +function isMySQL(k: Knex): boolean { + return String(k.client.config.client).includes('mysql') +} + +function normalized(sql: string): string { + return sql.replaceAll(/\s+/g, ' ').replaceAll(' IF NOT EXISTS ', ' ').trim() +} + +async function validateMysqlTable(k: Knex, table: string, keys: boolean, names: string[]): Promise { + const primary = keys ? names : ['snapshotTableId'] + const [columns]: Array< + Array<{ name: string; type: string; nullable: string; defaultValue: unknown; extra: string }> + > = await k.raw( + 'SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION', + [table] + ) + const [indexes]: Array< + Array<{ name: string; columnName: string; nonUnique: number; direction: string; prefix: unknown }> + > = await k.raw( + 'SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX', + [table] + ) + const types = keys ? ['int', 'int unsigned', 'int unsigned'] : ['int', 'int unsigned', 'tinyint'] + return ( + Array.isArray(columns) && + columns.length === names.length && + columns.every( + (column, i) => + column.name === names[i] && + column.type.replaceAll(/\(\d+\)/g, '') === types[i] && + column.nullable === 'NO' && + column.defaultValue === null && + column.extra === '' + ) && + Array.isArray(indexes) && + indexes.every(index => + index.name === 'PRIMARY' ? index.direction === 'A' && index.prefix === null : Number(index.nonUnique) === 1 + ) && + JSON.stringify(indexes.filter(index => index.name === 'PRIMARY').map(index => index.columnName)) === + JSON.stringify(primary) + ) +} + +async function validateSqlitePrimaryIndex(k: Knex, name: string | undefined, names: string[]): Promise { + if (name === undefined) return false + const parts: Array<{ name: string; desc: number; coll: string; key: number }> = await k.raw( + 'PRAGMA index_xinfo(??)', + [name] + ) + const indexed = parts.filter(part => part.key === 1) + return ( + indexed.length === names.length && + indexed.every((part, i) => part.name === names[i] && part.desc === 0 && part.coll === 'BINARY') + ) +} + +async function validateSqliteTable(k: Knex, table: string, keys: boolean, names: string[]): Promise { + const columns: Array<{ + name: string + type: string + notnull: number + dflt_value: unknown + pk: number + hidden: number + }> = await k.raw('PRAGMA table_xinfo(??)', [table]) + const indexes: Array<{ name: string; unique: number; origin: string; partial: number }> = await k.raw( + 'PRAGMA index_list(??)', + [table] + ) + const expectedPk = keys ? [1, 2, 3] : [1, 0, 0] + const nullable = keys ? [1, 1, 1] : [0, 1, 1] + const types = keys ? ['integer', 'integer', 'integer'] : ['integer', 'integer', 'boolean'] + const valid = + Array.isArray(columns) && + columns.length === names.length && + columns.every( + (column, i) => + column.name === names[i] && + column.type.toLowerCase() === types[i] && + column.notnull === nullable[i] && + column.dflt_value === null && + column.pk === expectedPk[i] && + column.hidden === 0 + ) && + Array.isArray(indexes) && + indexes.every(index => index.unique === 0 || (index.origin === 'pk' && index.partial === 0)) + if (!valid || !keys) return valid + return await validateSqlitePrimaryIndex(k, indexes.find(index => index.origin === 'pk')?.name, names) +} + +async function validateTable(k: Knex, table: string): Promise { + const keys = table === KEYS + const names = keys + ? ['snapshotTableId', 'snapshotUserId', 'snapshotRowId'] + : ['snapshotTableId', 'afterRowId', 'complete'] + const valid = isMySQL(k) + ? await validateMysqlTable(k, table, keys, names) + : await validateSqliteTable(k, table, keys, names) + if (!valid) throw new WERR_INVALID_OPERATION('Snapshot profile table definition mismatch') +} + +type TriggerEvent = 'DELETE' | 'UPDATE' | 'INSERT' + +export function snapshotProfileTriggerDefinition( + mysql: boolean, + table: string, + key: string, + tableId: number, + event: TriggerEvent +) { + const name = `snapshot_profile_${tableId}_${event.toLowerCase()}` + const remove = `DELETE FROM ${KEYS} WHERE snapshotTableId = ${tableId} AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.${key};` + const add = `INSERT INTO ${KEYS} (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (${tableId}, NEW.userId, NEW.${key});` + const changed = mysql + ? `NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.${key} <=> NEW.${key})` + : `OLD.userId IS NOT NEW.userId OR OLD.${key} IS NOT NEW.${key}` + const statements = { DELETE: remove, INSERT: add, UPDATE: remove + ' ' + add }[event] + const body = + mysql && event === 'UPDATE' ? `BEGIN IF ${changed} THEN ${statements} END IF; END` : `BEGIN ${statements} END` + let qualifier = '' + if (mysql) qualifier = ' FOR EACH ROW' + else if (event === 'UPDATE') qualifier = ` WHEN ${changed}` + return { name, body, sql: `CREATE TRIGGER ${name} AFTER ${event} ON ${table}${qualifier} ${body}` } +} + +async function mysqlTriggerExists( + k: Knex, + name: string, + table: string, + event: TriggerEvent, + body: string +): Promise { + const [rows]: Array> = await k.raw( + 'SELECT EVENT_MANIPULATION AS event, ACTION_TIMING AS timing, EVENT_OBJECT_TABLE AS tableName, ACTION_STATEMENT AS body FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = DATABASE() AND TRIGGER_NAME = ?', + [name] + ) + if (!Array.isArray(rows)) throw new WERR_INVALID_OPERATION('Invalid snapshot profile trigger metadata') + if (rows.length === 0) return false + const row = rows[0] + if ( + rows.length !== 1 || + row?.event !== event || + row.timing !== 'AFTER' || + row.tableName !== table || + normalized(row.body) !== normalized(body) + ) + throw new WERR_INVALID_OPERATION('Snapshot profile trigger definition mismatch') + return true +} + +async function sqliteTriggerExists(k: Knex, name: string, sql: string): Promise { + const row: { sql: string } | undefined = await k('sqlite_master').where({ type: 'trigger', name }).first('sql') + if (row === undefined) return false + if (normalized(row.sql) !== normalized(sql)) + throw new WERR_INVALID_OPERATION('Snapshot profile trigger definition mismatch') + return true +} + +async function installTrigger( + k: Knex, + table: string, + key: string, + tableId: number, + event: TriggerEvent, + create = true +): Promise { + const mysql = isMySQL(k) + const { name, body, sql } = snapshotProfileTriggerDefinition(mysql, table, key, tableId, event) + const exists = mysql ? await mysqlTriggerExists(k, name, table, event, body) : await sqliteTriggerExists(k, name, sql) + if (!exists && create) await k.raw(sql) +} + +async function bootstrapTable(k: Knex, table: string, key: string, tableId: number): Promise { + await k(PROGRESS) + .insert({ snapshotTableId: tableId, afterRowId: 0, complete: false }) + .onConflict('snapshotTableId') + .ignore() + let complete = false + function* unfinishedPages() { + while (!complete) yield undefined + } + await runInSeries(unfinishedPages(), async () => { + complete = await k.transaction(async trx => { + // SQLite must acquire its writer lock before reading a resumable position. + // MySQL takes a current row lock without loading an older read-view value. + if (!isMySQL(k)) + await trx(PROGRESS) + .where('snapshotTableId', tableId) + .update({ afterRowId: trx.ref('afterRowId') }) + const query = trx(PROGRESS).where('snapshotTableId', tableId) + if (isMySQL(k)) void query.forUpdate() + const state: { afterRowId: number; complete: boolean | number } | undefined = await query.first() + if ( + state === undefined || + !Number.isSafeInteger(state.afterRowId) || + state.afterRowId < 0 || + ![false, true, 0, 1].includes(state.complete) + ) { + throw new WERR_INVALID_OPERATION('Invalid snapshot profile bootstrap position') + } + if (state.complete === true || state.complete === 1) return true + const source = trx(table).select(key, 'userId').where(key, '>', state.afterRowId).orderBy(key).limit(PAGE_ROWS) + if (isMySQL(k)) void source.forUpdate() + const rows: Array> = await source + const keys = rows.map(row => { + const rowId = row[key] + const userId = row.userId + if ( + rowId === undefined || + userId === undefined || + !Number.isSafeInteger(rowId) || + rowId < 1 || + !Number.isSafeInteger(userId) || + userId < 1 + ) { + throw new WERR_INVALID_OPERATION('Invalid snapshot profile source key') + } + return { snapshotTableId: tableId, snapshotUserId: userId, snapshotRowId: rowId } + }) + if (keys.length !== 0) + await trx(KEYS).insert(keys).onConflict(['snapshotTableId', 'snapshotUserId', 'snapshotRowId']).ignore() + const finished = rows.length < PAGE_ROWS + await trx(PROGRESS) + .where('snapshotTableId', tableId) + .update({ afterRowId: keys.at(-1)?.snapshotRowId ?? state.afterRowId, complete: finished }) + return finished + }) + }) +} + +/** Add auxiliary keys without altering any standard-table index or OFFSET plan. */ +export async function addSnapshotProfileIndexes(k: Knex): Promise { + if (isMySQL(k) && k.isTransaction) + throw new WERR_INVALID_OPERATION('Snapshot profile migration requires independent DDL and bootstrap transactions') + if (!(await k.schema.hasTable(KEYS))) { + await k.schema.createTable(KEYS, t => { + t.integer('snapshotTableId').notNullable() + t.integer('snapshotUserId').unsigned().notNullable() + t.integer('snapshotRowId').unsigned().notNullable() + t.primary(['snapshotTableId', 'snapshotUserId', 'snapshotRowId']) + }) + } + await validateTable(k, KEYS) + if (!(await k.schema.hasTable(PROGRESS))) { + await k.schema.createTable(PROGRESS, t => { + t.integer('snapshotTableId').primary() + t.integer('afterRowId').unsigned().notNullable() + t.boolean('complete').notNullable() + }) + } + await validateTable(k, PROGRESS) + await runInSeries(snapshotProfileTables.entries(), async ([tableId, { table, key }]) => { + // Deletion must be observed before any partial installation can add keys. + await installTrigger(k, table, key, tableId, 'DELETE') + await installTrigger(k, table, key, tableId, 'UPDATE') + await installTrigger(k, table, key, tableId, 'INSERT') + await bootstrapTable(k, table, key, tableId) + }) +} + +/** Call only after snapshot readers are drained; standard rows remain intact. */ +export async function removeSnapshotProfileIndexes(k: Knex): Promise { + if (isMySQL(k) && k.isTransaction) + throw new WERR_INVALID_OPERATION('Snapshot profile migration requires independent DDL and bootstrap transactions') + if (await k.schema.hasTable(KEYS)) await validateTable(k, KEYS) + if (await k.schema.hasTable(PROGRESS)) await validateTable(k, PROGRESS) + await runInSeries(snapshotProfileTables.entries(), async ([tableId, { table, key }]) => { + await runInSeries(['INSERT', 'UPDATE', 'DELETE'] as const, async event => { + await installTrigger(k, table, key, tableId, event, false) + }) + }) + await runInSeries(snapshotProfileTables.entries(), async ([tableId]) => { + // Stop every producer before removing the last deletion observer. + await runInSeries(['insert', 'update', 'delete'], async event => { + await k.raw(`DROP TRIGGER IF EXISTS snapshot_profile_${tableId}_${event}`) + }) + }) + await k.schema.dropTableIfExists(PROGRESS) + await k.schema.dropTableIfExists(KEYS) +} + +/** Read only within the retained view that owns this snapshot's header and pages. */ +export async function readSnapshotProfileIndexState(k: Knex, config?: Knex.MigratorConfig): Promise { + const tableName = config?.tableName ?? 'knex_migrations' + const journalSchema = k.schema + if (config?.schemaName !== undefined) void journalSchema.withSchema(config.schemaName) + if (!(await journalSchema.hasTable(tableName))) return false + const journal = k(tableName).where('name', SNAPSHOT_PROFILE_INDEX_MIGRATION) + if (config?.schemaName !== undefined) void journal.withSchema(config.schemaName) + if ((await journal.first('name')) === undefined) return false + if (!(await k.schema.hasTable(KEYS)) || !(await k.schema.hasTable(PROGRESS))) { + throw new WERR_INVALID_OPERATION('Snapshot profile index migration is incomplete') + } + await validateTable(k, KEYS) + await validateTable(k, PROGRESS) + const rows: Array<{ snapshotTableId: number; afterRowId: number; complete: unknown }> = await k(PROGRESS) + .select('snapshotTableId', 'afterRowId', 'complete') + .orderBy('snapshotTableId') + .limit(snapshotProfileTables.length + 1) + if ( + rows.length !== snapshotProfileTables.length || + rows.some( + (row, index) => + row.snapshotTableId !== index || + !Number.isSafeInteger(row.afterRowId) || + row.afterRowId < 0 || + (row.complete !== true && row.complete !== 1) + ) + ) + throw new WERR_INVALID_OPERATION('Snapshot profile index migration is incomplete') + return true +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotRelationIndexMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotRelationIndexMigration.ts new file mode 100644 index 000000000..aa4b99387 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotRelationIndexMigration.ts @@ -0,0 +1,519 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { runInSeries } from '../../utility/runInSeries' + +export const SNAPSHOT_RELATION_INDEX_MIGRATION = '2026-10-01-004 add snapshot relation key indexes' + +export const snapshotNumericRelations = [ + { table: 'tx_labels_map', left: 'tx_labels', leftKey: 'txLabelId', right: 'transactions', rightKey: 'transactionId' }, + { table: 'output_tags_map', left: 'output_tags', leftKey: 'outputTagId', right: 'outputs', rightKey: 'outputId' } +] as const + +type Relation = (typeof snapshotNumericRelations)[number] +type Side = 'left' | 'right' +type Event = 'INSERT' | 'UPDATE' | 'DELETE' +interface Trigger { + name: string + table: string + timing: 'BEFORE' | 'AFTER' + event: Event + body: string + sql: string +} + +const KEYS = 'snapshot_relation_keys' +const PROGRESS = 'snapshot_relation_index_progress' +const PAGE_ROWS = 256 +const keyColumns = ['snapshotTableId', 'snapshotUserId', 'snapshotLeftId', 'snapshotRightId'] +const indexes = [ + { + name: 'snapshot_relation_right', + columns: ['snapshotTableId', 'snapshotUserId', 'snapshotRightId', 'snapshotLeftId'] + }, + { name: 'snapshot_relation_map', columns: ['snapshotTableId', 'snapshotLeftId', 'snapshotRightId', 'snapshotUserId'] } +] + +function mysql(k: Knex): boolean { + return String(k.client.config.client).includes('mysql') +} + +function normalized(sql: string): string { + return sql.replaceAll(/\s+/g, ' ').trim() +} + +function sideInfo(relation: Relation, side: Side) { + return side === 'left' + ? { table: relation.left, key: relation.leftKey, index: 'snapshotLeftId', bit: 1 } + : { table: relation.right, key: relation.rightKey, index: 'snapshotRightId', bit: 2 } +} + +function insertMembership(isMysql: boolean, select: string, bit: number): string { + const merge = isMysql + ? ` ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | ${bit}` + : ` ON CONFLICT(${keyColumns.join(', ')}) DO UPDATE SET snapshotMembership = snapshotMembership | ${bit}` + return `INSERT INTO ${KEYS} (${keyColumns.join(', ')}, snapshotMembership) ${select}${merge};` +} + +function addMapSide(isMysql: boolean, relation: Relation, tableId: number, side: Side): string { + const { table, key, bit } = sideInfo(relation, side) + // An explicit locking read is required even under READ COMMITTED. The parent + // owner must stay current until this map transaction publishes its keys. + const lock = isMysql ? ' FOR SHARE' : '' + return insertMembership( + isMysql, + `SELECT ${tableId}, userId, NEW.${relation.leftKey}, NEW.${relation.rightKey}, ${bit} FROM ${table} WHERE ${key} = NEW.${key}${lock}`, + bit + ) +} + +function removeMap(relation: Relation, tableId: number): string { + return `DELETE FROM ${KEYS} WHERE snapshotTableId = ${tableId} AND snapshotLeftId = OLD.${relation.leftKey} AND snapshotRightId = OLD.${relation.rightKey};` +} + +function removeParent(relation: Relation, tableId: number, side: Side): string { + const { key, index, bit } = sideInfo(relation, side) + const where = `snapshotTableId = ${tableId} AND snapshotUserId = OLD.userId AND ${index} = OLD.${key}` + return `UPDATE ${KEYS} SET snapshotMembership = snapshotMembership & ${3 ^ bit} WHERE ${where}; DELETE FROM ${KEYS} WHERE ${where} AND snapshotMembership = 0;` +} + +function addParent(isMysql: boolean, relation: Relation, tableId: number, side: Side): string { + const { key, bit } = sideInfo(relation, side) + const lock = isMysql ? ' FOR SHARE' : '' + const select = `SELECT ${tableId}, NEW.userId, ${relation.leftKey}, ${relation.rightKey}, ${bit} FROM ${relation.table} WHERE ${key} = NEW.${key} ORDER BY ${relation.leftKey}, ${relation.rightKey}${lock}` + return insertMembership(isMysql, select, bit) +} + +function trigger( + isMysql: boolean, + name: string, + table: string, + timing: Trigger['timing'], + event: Event, + statements: string, + changed?: string +): Trigger { + const body = + isMysql && changed !== undefined ? `BEGIN IF ${changed} THEN ${statements} END IF; END` : `BEGIN ${statements} END` + let qualifier = '' + if (isMysql) qualifier = ' FOR EACH ROW' + else if (changed !== undefined) qualifier = ` WHEN ${changed}` + return { + name, + table, + timing, + event, + body, + sql: `CREATE TRIGGER ${name} ${timing} ${event} ON ${table}${qualifier} ${body}` + } +} + +export function snapshotRelationIndexTriggers(isMysql: boolean): { observers: Trigger[]; producers: Trigger[] } { + const observers: Trigger[] = [] + const producers: Trigger[] = [] + const different = (key: string): string => + isMysql ? `NOT (OLD.${key} <=> NEW.${key})` : `OLD.${key} IS NOT NEW.${key}` + for (const [tableId, relation] of snapshotNumericRelations.entries()) { + const prefix = `snapshot_relation_${tableId}` + const changed = `${different(relation.leftKey)} OR ${different(relation.rightKey)}` + const remove = removeMap(relation, tableId) + const add = addMapSide(isMysql, relation, tableId, 'left') + ' ' + addMapSide(isMysql, relation, tableId, 'right') + observers.push( + trigger(isMysql, `${prefix}_map_delete`, relation.table, 'AFTER', 'DELETE', remove), + trigger(isMysql, `${prefix}_map_before_update`, relation.table, 'BEFORE', 'UPDATE', remove, changed) + ) + producers.push( + trigger(isMysql, `${prefix}_map_insert`, relation.table, 'AFTER', 'INSERT', add), + trigger(isMysql, `${prefix}_map_after_update`, relation.table, 'AFTER', 'UPDATE', add, changed) + ) + for (const side of ['left', 'right'] as const) { + const { table, key } = sideInfo(relation, side) + const ownerChanged = `${different(key)} OR ${different('userId')}` + const subtract = removeParent(relation, tableId, side) + const append = addParent(isMysql, relation, tableId, side) + observers.push( + trigger(isMysql, `${prefix}_${side}_delete`, table, 'AFTER', 'DELETE', subtract), + trigger(isMysql, `${prefix}_${side}_before_update`, table, 'BEFORE', 'UPDATE', subtract, ownerChanged) + ) + producers.push( + trigger(isMysql, `${prefix}_${side}_insert`, table, 'AFTER', 'INSERT', append), + trigger(isMysql, `${prefix}_${side}_after_update`, table, 'AFTER', 'UPDATE', append, ownerChanged) + ) + } + } + return { observers, producers } +} + +async function validateTrigger(k: Knex, expected: Trigger): Promise { + if (!mysql(k)) { + const row: { sql: string } | undefined = await k('sqlite_master') + .where({ type: 'trigger', name: expected.name }) + .first('sql') + if (row === undefined) return false + if (normalized(row.sql) !== normalized(expected.sql)) + throw new WERR_INVALID_OPERATION('Snapshot relation trigger definition mismatch') + return true + } + const [rows]: Array> = await k.raw( + 'SELECT EVENT_MANIPULATION AS event, ACTION_TIMING AS timing, EVENT_OBJECT_TABLE AS tableName, ACTION_STATEMENT AS body FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = DATABASE() AND TRIGGER_NAME = ?', + [expected.name] + ) + if (!Array.isArray(rows)) throw new WERR_INVALID_OPERATION('Invalid snapshot relation trigger metadata') + if (rows.length === 0) return false + const row = rows[0] + if ( + rows.length !== 1 || + row?.event !== expected.event || + row.timing !== expected.timing || + row.tableName !== expected.table || + normalized(row.body) !== normalized(expected.body) + ) { + throw new WERR_INVALID_OPERATION('Snapshot relation trigger definition mismatch') + } + return true +} + +interface Column { + name: string + type: 'integer' | 'boolean' + unsigned?: boolean + primary: number +} + +function tableColumns(keys: boolean): Column[] { + if (keys) + return [ + { name: 'snapshotTableId', type: 'integer', primary: 1 }, + { name: 'snapshotUserId', type: 'integer', unsigned: true, primary: 2 }, + { name: 'snapshotLeftId', type: 'integer', unsigned: true, primary: 3 }, + { name: 'snapshotRightId', type: 'integer', unsigned: true, primary: 4 }, + { name: 'snapshotMembership', type: 'integer', unsigned: true, primary: 0 } + ] + return [ + { name: 'snapshotTableId', type: 'integer', primary: 1 }, + { name: 'afterLeftId', type: 'integer', unsigned: true, primary: 0 }, + { name: 'afterRightId', type: 'integer', unsigned: true, primary: 0 }, + { name: 'complete', type: 'boolean', primary: 0 } + ] +} + +async function sqliteIndex(k: Knex, name: string, columns: string[]): Promise { + const rows: Array<{ name: string; desc: number; coll: string; key: number }> = await k.raw('PRAGMA index_xinfo(??)', [ + name + ]) + const parts = rows.filter(row => row.key === 1) + return ( + parts.length === columns.length && + parts.every((row, i) => row.name === columns[i] && row.desc === 0 && row.coll === 'BINARY') + ) +} + +async function sqliteTable(k: Knex, table: string, keys: boolean, secondary: boolean): Promise { + const expected = tableColumns(keys) + const columns: Array<{ + name: string + type: string + notnull: number + dflt_value: unknown + pk: number + hidden: number + }> = await k.raw('PRAGMA table_xinfo(??)', [table]) + if ( + !Array.isArray(columns) || + columns.length !== expected.length || + columns.some((column, i) => { + const field = expected[i] + return ( + column.name !== field.name || + column.type.toLowerCase() !== field.type || + column.notnull !== (!keys && i === 0 ? 0 : 1) || + column.dflt_value !== null || + column.pk !== field.primary || + column.hidden !== 0 + ) + }) + ) + return false + const existing: Array<{ name: string; unique: number; origin: string; partial: number }> = await k.raw( + 'PRAGMA index_list(??)', + [table] + ) + if (existing.some(index => index.unique !== 0 && (index.origin !== 'pk' || index.partial !== 0))) return false + if (!keys) return true + const primary = existing.find(index => index.origin === 'pk') + if (primary === undefined || !(await sqliteIndex(k, primary.name, keyColumns))) return false + if (!secondary) return true + for (const expectedIndex of indexes) { + const found = existing.find(index => index.name === expectedIndex.name) + if (found?.unique !== 0 || found.partial !== 0 || !(await sqliteIndex(k, found.name, expectedIndex.columns))) + return false + } + return true +} + +async function mysqlTable(k: Knex, table: string, keys: boolean, secondary: boolean): Promise { + const expected = tableColumns(keys) + const [columns]: Array< + Array<{ name: string; type: string; nullable: string; defaultValue: unknown; extra: string }> + > = await k.raw( + 'SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION', + [table] + ) + if ( + !Array.isArray(columns) || + columns.length !== expected.length || + columns.some((column, i) => { + const field = expected[i] + const integerType = field.unsigned === true ? 'int unsigned' : 'int' + const type = field.type === 'boolean' ? 'tinyint' : integerType + return ( + column.name !== field.name || + column.type.replaceAll(/\(\d+\)/g, '') !== type || + column.nullable !== 'NO' || + column.defaultValue !== null || + column.extra !== '' + ) + }) + ) + return false + const [parts]: Array< + Array<{ name: string; columnName: string; nonUnique: number; direction: string; prefix: unknown }> + > = await k.raw( + 'SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX', + [table] + ) + if (!Array.isArray(parts) || parts.some(index => index.name !== 'PRIMARY' && Number(index.nonUnique) !== 1)) + return false + const required = [ + { name: 'PRIMARY', columns: keys ? keyColumns : ['snapshotTableId'] }, + ...(keys && secondary ? indexes : []) + ] + return required.every(index => { + const found = parts.filter(part => part.name === index.name) + return ( + found.length === index.columns.length && + found.every((part, i) => part.columnName === index.columns[i] && part.direction === 'A' && part.prefix === null) + ) + }) +} + +async function validateTable(k: Knex, table: string, secondary = true): Promise { + const valid = mysql(k) + ? await mysqlTable(k, table, table === KEYS, secondary) + : await sqliteTable(k, table, table === KEYS, secondary) + if (!valid) throw new WERR_INVALID_OPERATION('Snapshot relation table definition mismatch') +} + +async function ensureTables(k: Knex): Promise { + if (!(await k.schema.hasTable(KEYS))) { + await k.schema.createTable(KEYS, t => { + t.integer('snapshotTableId').notNullable() + t.integer('snapshotUserId').unsigned().notNullable() + t.integer('snapshotLeftId').unsigned().notNullable() + t.integer('snapshotRightId').unsigned().notNullable() + t.integer('snapshotMembership').unsigned().notNullable() + t.primary(keyColumns) + }) + } + // MySQL DDL commits independently. Resume an interrupted creation between + // the table and either auxiliary index without trusting a conflicting index. + await validateTable(k, KEYS, false) + await runInSeries(indexes, async index => { + const exists = mysql(k) + ? ( + await k.raw( + 'SELECT INDEX_NAME FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND INDEX_NAME = ? LIMIT 1', + [KEYS, index.name] + ) + )[0].length !== 0 + : (await k('sqlite_master').where({ type: 'index', name: index.name }).first('name')) !== undefined + if (!exists) + await k.schema.alterTable(KEYS, t => { + t.index(index.columns, index.name) + }) + }) + await validateTable(k, KEYS) + if (!(await k.schema.hasTable(PROGRESS))) { + await k.schema.createTable(PROGRESS, t => { + t.integer('snapshotTableId').primary() + t.integer('afterLeftId').unsigned().notNullable() + t.integer('afterRightId').unsigned().notNullable() + t.boolean('complete').notNullable() + }) + } + await validateTable(k, PROGRESS) +} + +interface Position { + afterLeftId: number + afterRightId: number + complete: boolean | number +} + +function validPosition(state: Position | undefined): state is Position { + return ( + state !== undefined && + Number.isSafeInteger(state.afterLeftId) && + state.afterLeftId >= 0 && + Number.isSafeInteger(state.afterRightId) && + state.afterRightId >= 0 && + [false, true, 0, 1].includes(state.complete) + ) +} + +function positive(value: number | undefined): number { + if (value === undefined || !Number.isSafeInteger(value) || value < 1) + throw new WERR_INVALID_OPERATION('Invalid snapshot relation source key') + return value +} + +async function bootstrapPage(k: Knex, relation: Relation, tableId: number): Promise { + return await k.transaction(async trx => { + if (!mysql(k)) + await trx(PROGRESS) + .where('snapshotTableId', tableId) + .update({ afterLeftId: trx.ref('afterLeftId') }) + const progress = trx(PROGRESS).where('snapshotTableId', tableId) + if (mysql(k)) void progress.forUpdate() + const state: Position | undefined = await progress.first() + if (!validPosition(state)) throw new WERR_INVALID_OPERATION('Invalid snapshot relation bootstrap position') + if (state.complete === true || state.complete === 1) return true + const source = trx(relation.table) + .select(relation.leftKey, relation.rightKey) + .orderBy([relation.leftKey, relation.rightKey]) + .limit(PAGE_ROWS) + if (mysql(k)) { + // InnoDB does not reliably range-optimize the equivalent tuple inequality. + void source + .whereRaw('(?? > ? OR (?? = ? AND ?? > ?))', [ + relation.leftKey, + state.afterLeftId, + relation.leftKey, + state.afterLeftId, + relation.rightKey, + state.afterRightId + ]) + .forUpdate() + } else { + void source.whereRaw('(??, ??) > (?, ?)', [ + relation.leftKey, + relation.rightKey, + state.afterLeftId, + state.afterRightId + ]) + } + const rows: Array> = await source + for (const row of rows) { + positive(row[relation.leftKey]) + positive(row[relation.rightKey]) + } + await runInSeries(['left', 'right'] as const, async side => { + const { table, key, bit } = sideInfo(relation, side) + const ids = [...new Set(rows.map(row => row[key]))].sort((a, b) => a - b) + if (ids.length === 0) return + const query = trx(table).select(key, 'userId').whereIn(key, ids).orderBy(key) + if (mysql(k)) void query.forShare() + const parents: Array> = await query + const owners = new Map(parents.map(row => [positive(row[key]), positive(row.userId)])) + const memberships = rows.flatMap(row => { + const owner = owners.get(row[key]) + return owner === undefined + ? [] + : [ + { + snapshotTableId: tableId, + snapshotUserId: owner, + snapshotLeftId: row[relation.leftKey], + snapshotRightId: row[relation.rightKey], + snapshotMembership: bit + } + ] + }) + if (memberships.length !== 0) + await trx(KEYS) + .insert(memberships) + .onConflict(keyColumns) + .merge({ snapshotMembership: trx.raw('?? | ?', ['snapshotMembership', bit]) }) + }) + const last = rows.at(-1) + const complete = rows.length < PAGE_ROWS + await trx(PROGRESS) + .where('snapshotTableId', tableId) + .update({ + afterLeftId: last?.[relation.leftKey] ?? state.afterLeftId, + afterRightId: last?.[relation.rightKey] ?? state.afterRightId, + complete + }) + return complete + }) +} + +/** Keep standard schema and cursor order intact while indexing OR ownership. */ +export async function addSnapshotRelationIndexes(k: Knex): Promise { + if (mysql(k) && k.isTransaction) + throw new WERR_INVALID_OPERATION('Snapshot relation migration requires independent DDL and bootstrap transactions') + await ensureTables(k) + const { observers, producers } = snapshotRelationIndexTriggers(mysql(k)) + // Every graph-wide removal observer precedes every possible producer. + await runInSeries([...observers, ...producers], async expected => { + if (!(await validateTrigger(k, expected))) await k.raw(expected.sql) + }) + await runInSeries(snapshotNumericRelations.entries(), async ([tableId, relation]) => { + await k(PROGRESS) + .insert({ snapshotTableId: tableId, afterLeftId: 0, afterRightId: 0, complete: false }) + .onConflict('snapshotTableId') + .ignore() + let complete = false + function* unfinishedPages() { + while (!complete) yield undefined + } + await runInSeries(unfinishedPages(), async () => { + complete = await bootstrapPage(k, relation, tableId) + }) + }) +} + +/** Readers must be drained before removal; standard rows remain intact. */ +export async function removeSnapshotRelationIndexes(k: Knex): Promise { + if (mysql(k) && k.isTransaction) + throw new WERR_INVALID_OPERATION('Snapshot relation migration requires independent DDL and bootstrap transactions') + if (await k.schema.hasTable(KEYS)) await validateTable(k, KEYS) + if (await k.schema.hasTable(PROGRESS)) await validateTable(k, PROGRESS) + const { observers, producers } = snapshotRelationIndexTriggers(mysql(k)) + const ordered = [...producers, ...observers] + await runInSeries(ordered, async expected => { + await validateTrigger(k, expected) + }) + await runInSeries(ordered, async expected => { + await k.raw('DROP TRIGGER IF EXISTS ??', [expected.name]) + }) + await k.schema.dropTableIfExists(PROGRESS) + await k.schema.dropTableIfExists(KEYS) +} + +/** Read inside the same retained view as the snapshot header and table pages. */ +export async function readSnapshotRelationIndexState(k: Knex, config?: Knex.MigratorConfig): Promise { + const tableName = config?.tableName ?? 'knex_migrations' + const schema = k.schema + if (config?.schemaName !== undefined) void schema.withSchema(config.schemaName) + if (!(await schema.hasTable(tableName))) return false + const journal = k(tableName).where('name', SNAPSHOT_RELATION_INDEX_MIGRATION) + if (config?.schemaName !== undefined) void journal.withSchema(config.schemaName) + if ((await journal.first('name')) === undefined) return false + if (!(await k.schema.hasTable(KEYS)) || !(await k.schema.hasTable(PROGRESS))) + throw new WERR_INVALID_OPERATION('Snapshot relation index migration is incomplete') + await validateTable(k, KEYS) + await validateTable(k, PROGRESS) + const states: Array = await k(PROGRESS) + .select('snapshotTableId', 'afterLeftId', 'afterRightId', 'complete') + .orderBy('snapshotTableId') + .limit(snapshotNumericRelations.length + 1) + if ( + states.length !== snapshotNumericRelations.length || + states.some( + (state, i) => + state.snapshotTableId !== i || !validPosition(state) || (state.complete !== true && state.complete !== 1) + ) + ) + throw new WERR_INVALID_OPERATION('Snapshot relation index migration is incomplete') + return true +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqlMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqlMigration.ts new file mode 100644 index 000000000..4c79d180b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqlMigration.ts @@ -0,0 +1,12 @@ +import type { Knex } from 'knex' + +/** Snapshot auxiliary schemas currently require SQLite or MySQL. Postgres keeps + * its standard wallet schema and legacy sync path. A future Postgres snapshot + * implementation must use new forward migrations, rather than changing these + * already-recorded no-op entries. */ +export function forSnapshotSqlDialect(run: (k: Knex) => Promise): (k: Knex) => Promise { + return async k => { + if (k.client.dialect === 'postgresql') return + await run(k) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIdentity.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIdentity.ts new file mode 100644 index 000000000..f5a81600d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIdentity.ts @@ -0,0 +1,130 @@ +import { runInSeries } from '../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { Knex } from 'knex' +// Metadata-bound conflict witnesses are maintained by the additive SQLite generation. +export interface SourceIdentity { + table: string + key: string + owner?: string +} +interface Part { + name: string + collation: string +} +interface Column { + name: string + type: string + pk: number + hidden: number +} +export interface IdentityDefinition { + source: SourceIdentity + table: string + columns: Column[] + unique: Part[][] +} +const quote = (name: string): string => '"' + name.replaceAll('"', '""') + '"' +export async function readIdentity(k: Knex, source: SourceIdentity): Promise { + const columns: Column[] = await k.raw('PRAGMA table_xinfo(??)', [source.table]) + const primary = columns.filter(column => column.pk !== 0) + if (primary.length !== 1 || primary[0].name !== source.key || primary[0].type.toLowerCase() !== 'integer') + throw new WERR_INVALID_OPERATION('Unsupported numeric identity') + const indexes: Array<{ + name: string + unique: number + partial: number + }> = await k.raw('PRAGMA index_list(??)', [source.table]) + const unique: Part[][] = [] + await runInSeries( + indexes.filter(index => index.unique !== 0), + async index => { + if (index.partial !== 0) throw new WERR_INVALID_OPERATION('Unsupported partial unique identity') + const metadata: Array<{ + name: string | null + coll: string + key: number + }> = await k.raw('PRAGMA index_xinfo(??)', [index.name]) + const parts = metadata.filter(part => part.key !== 0) + if ( + parts.length === 0 || + parts.some(part => part.name === null || !['BINARY', 'NOCASE', 'RTRIM'].includes(part.coll)) + ) + throw new WERR_INVALID_OPERATION('Unsupported unique identity comparison') + unique.push(parts.map(part => ({ name: part.name!, collation: part.coll }))) + } + ) + const names = new Set([ + source.key, + ...(source.owner ? [source.owner] : []), + ...unique.flatMap(parts => parts.map(part => part.name)) + ]) + const selected = [...names].map(name => columns.find(column => column.name === name)) + if ( + selected.some( + column => column?.hidden !== 0 || !/^(integer|bigint|boolean|varchar\([1-9]\d*\))$/i.test(column.type) + ) + ) + throw new WERR_INVALID_OPERATION('Unsupported identity column') + return { + source, + table: 'snapshot_identity_' + source.table, + columns: selected as Column[], + unique + } +} +export function identityDDL(identity: IdentityDefinition): string[] { + const primary = [identity.source.key, ...(identity.source.owner ? [identity.source.owner] : [])] + const fields = identity.columns.map( + column => `${quote(column.name)} ${column.type}${primary.includes(column.name) ? ' NOT NULL' : ''}` + ) + return [ + `CREATE TABLE ${quote(identity.table)} (${fields.join(',')},PRIMARY KEY (${primary.map(quote).join(',')}))`, + ...identity.unique.map( + (parts, index) => + `CREATE INDEX ${quote(identity.table + '_' + index)} ON ${quote(identity.table)} (${parts.map(part => quote(part.name) + ' COLLATE ' + part.collation).join(',')})` + ) + ] +} +/** SQL unique equality deliberately excludes NULL, as the source constraint does. */ +function conflicts(identity: IdentityDefinition, updated: boolean): string { + const key = quote(identity.source.key) + return [ + `${key}=NEW.${key}`, + ...(updated ? [`${key}=OLD.${key}`] : []), + ...identity.unique.map( + parts => + '(' + + parts.map(part => `${quote(part.name)} COLLATE ${part.collation}=NEW.${quote(part.name)}`).join(' AND ') + + ')' + ) + ].join(' OR ') +} +function merge(identity: IdentityDefinition, selection: string): string { + const columns = identity.columns.map(column => quote(column.name)) + const changed = columns.map(column => `${column} IS NOT excluded.${column}`).join(' OR ') + // A nested source trigger can expose a new owner before the outer trigger + // consumes the old one. Preserve both until membership cleanup runs. + const primary = [identity.source.key, ...(identity.source.owner ? [identity.source.owner] : [])] + const assignments = columns.map(column => `${column}=excluded.${column}`).join(',') + return `INSERT INTO ${quote(identity.table)} (${columns.join(',')}) ${selection} ON CONFLICT(${primary.map(quote).join(',')}) DO UPDATE SET ${assignments} WHERE ${changed};` +} +/** Observe existing rows only. An ignored source write cannot remove ownership. */ +export function observeIdentity(identity: IdentityDefinition, updated: boolean): string { + const columns = identity.columns.map(column => quote(column.name)).join(',') + return merge(identity, `SELECT ${columns} FROM ${quote(identity.source.table)} WHERE ${conflicts(identity, updated)}`) +} +/** Evaluated after the successful source operation, before the witness changes. */ +export function displacedIdentity(identity: IdentityDefinition, updated: boolean): string { + return `SELECT ${quote(identity.source.key)}${identity.source.owner ? ',' + quote(identity.source.owner) : ''} FROM ${quote(identity.table)} WHERE ${conflicts(identity, updated)}` +} +/** Run only after all affected membership families consume displacedIdentity. */ +export function finishIdentity(identity: IdentityDefinition, event: 'INSERT' | 'UPDATE' | 'DELETE'): string { + const key = quote(identity.source.key) + const row = event === 'DELETE' ? 'OLD' : 'NEW' + const remove = event === 'DELETE' ? `${key}=OLD.${key}` : conflicts(identity, event === 'UPDATE') + const columns = identity.columns.map(column => quote(column.name)).join(',') + return ( + `DELETE FROM ${quote(identity.table)} WHERE ${remove}; ` + + merge(identity, `SELECT ${columns} FROM ${quote(identity.source.table)} WHERE ${key}=${row}.${key}`) + ) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexBootstrap.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexBootstrap.ts new file mode 100644 index 000000000..b55bb7d71 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexBootstrap.ts @@ -0,0 +1,243 @@ +import { runInSeries } from '../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { Knex } from 'knex' +import { profiles, numeric, relations } from './snapshotSqliteMembership' +import { names, progress, metadata, validateInstalled, type Plan } from './snapshotSqliteIndexGeneration' +export interface Position { + stream: number + afterId: number + afterSecond: number + afterText: string + complete: number +} +const id = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) > 0 +const nonnegative = (value: unknown): value is number => Number.isSafeInteger(value) && Number(value) >= 0 +export function valid(state: Position): boolean { + if (state.stream !== 10 && state.afterText !== '') return false + if (state.stream !== 8 && state.stream !== 9 && state.afterSecond !== 0) return false + if ((state.stream === 8 || state.stream === 9) && (state.afterId === 0) !== (state.afterSecond === 0)) return false + if (state.stream === 10 && state.afterId === 0 && state.afterText !== '') return false + return ( + Number.isSafeInteger(state.stream) && + state.stream >= 0 && + state.stream < 12 && + nonnegative(state.afterId) && + nonnegative(state.afterSecond) && + typeof state.afterText === 'string' && + Array.from(state.afterText).length <= 100 && + Buffer.byteLength(state.afterText) <= 400 && + [0, 1].includes(state.complete) + ) +} +async function addProfile(trx: Knex, stream: number, state: Position): Promise> { + const { table, key } = numeric[stream] + const query = trx(table).select({ rowId: key }, 'userId').orderBy(key).limit(256) + if (state.afterId !== 0) void query.where(key, '>', state.afterId) + const rows: Array<{ + rowId: number + userId: number + }> = await query + if (rows.some(row => !id(row.rowId) || !id(row.userId))) + throw new WERR_INVALID_OPERATION('Invalid profile source identity') + if (rows.length) + await trx(names.profile) + .insert( + rows.map(row => ({ + snapshotTableId: stream, + snapshotUserId: row.userId, + snapshotRowId: row.rowId + })) + ) + .onConflict(['snapshotTableId', 'snapshotUserId', 'snapshotRowId']) + .ignore() + return { afterId: rows.at(-1)?.rowId ?? state.afterId, complete: rows.length < 256 ? 1 : 0 } +} +async function addRelations(trx: Knex, stream: number, state: Position): Promise> { + const relation = relations[stream - 8] + const query = trx(relation.table) + .select({ leftId: relation.leftKey, rightId: relation.rightKey }) + .orderBy([relation.leftKey, relation.rightKey]) + .limit(256) + if (state.afterId !== 0) + void query.whereRaw('(??,??)>(?,?)', [relation.leftKey, relation.rightKey, state.afterId, state.afterSecond]) + const rows: Array<{ + leftId: number + rightId: number + }> = await query + if (rows.some(row => !id(row.leftId) || !id(row.rightId))) + throw new WERR_INVALID_OPERATION('Invalid relation source identity') + await runInSeries(rows, async row => { + await runInSeries(['left', 'right'] as const, async side => { + const key = side === 'left' ? relation.leftKey : relation.rightKey + const bit = side === 'left' ? 1 : 2 + const owner = await trx(relation[side]) + .select('userId') + .where(key, side === 'left' ? row.leftId : row.rightId) + .first() + if (owner === undefined) return + if (!id(owner.userId)) throw new WERR_INVALID_OPERATION('Invalid relation source owner') + await trx(names.relation) + .insert({ + snapshotTableId: stream - 8, + snapshotUserId: owner.userId, + snapshotLeftId: row.leftId, + snapshotRightId: row.rightId, + snapshotMembership: bit + }) + .onConflict(['snapshotTableId', 'snapshotUserId', 'snapshotLeftId', 'snapshotRightId']) + .merge({ snapshotMembership: trx.raw('snapshotMembership | ?', [bit]) }) + }) + }) + return { + afterId: rows.at(-1)?.leftId ?? state.afterId, + afterSecond: rows.at(-1)?.rightId ?? state.afterSecond, + complete: rows.length < 256 ? 1 : 0 + } +} +async function addFields(trx: Knex, state: Position): Promise> { + const query = trx('certificate_fields') + .select( + 'certificateId', + 'userId', + trx.raw('CASE WHEN length(CAST(fieldName AS BLOB))<=400 THEN fieldName END AS fieldName'), + trx.raw('length(CAST(fieldName AS BLOB)) AS bytes') + ) + .orderBy(['fieldName', 'certificateId']) + .limit(256) + if (state.afterId !== 0) void query.whereRaw('(fieldName,certificateId)>(?,?)', [state.afterText, state.afterId]) + const rows: Array<{ + fieldName: string | null + certificateId: number + userId: number + bytes: number + }> = await query + if ( + rows.some( + row => + typeof row.fieldName !== 'string' || + Array.from(row.fieldName).length > 100 || + Buffer.byteLength(row.fieldName) !== row.bytes || + !id(row.certificateId) || + !id(row.userId) + ) + ) + throw new WERR_INVALID_OPERATION('Invalid certificate source identity') + await runInSeries(rows, async row => { + const parent = await trx('certificates').where('certificateId', row.certificateId).first('userId') + const owners = [ + { userId: row.userId, bit: 1 }, + ...(parent === undefined ? [] : [{ userId: parent.userId, bit: 2 }]) + ] + await runInSeries(owners, async owner => { + if (!id(owner.userId)) throw new WERR_INVALID_OPERATION('Invalid certificate source owner') + await trx(names.certificate) + .insert({ + snapshotUserId: owner.userId, + snapshotFieldName: row.fieldName, + snapshotCertificateId: row.certificateId, + snapshotMembership: owner.bit + }) + .onConflict(['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId']) + .merge({ snapshotMembership: trx.raw('snapshotMembership | ?', [owner.bit]) }) + }) + }) + return { + afterText: rows.at(-1)?.fieldName ?? state.afterText, + afterId: rows.at(-1)?.certificateId ?? state.afterId, + complete: rows.length < 256 ? 1 : 0 + } +} +async function addGlobal(trx: Knex, state: Position): Promise> { + const query = trx('transactions') + .select( + 'transactionId', + 'userId', + 'provenTxId', + trx.raw('CASE WHEN length(CAST(txid AS BLOB))<=256 THEN txid END AS txid'), + trx.raw('length(CAST(txid AS BLOB)) AS bytes') + ) + .orderBy('transactionId') + .limit(256) + if (state.afterId !== 0) void query.where('transactionId', '>', state.afterId) + const rows: Array<{ + transactionId: number + userId: number + provenTxId: number | null + txid: string | null + bytes: number | null + }> = await query + await runInSeries(rows, async row => { + if ( + !id(row.transactionId) || + !id(row.userId) || + (row.provenTxId !== null && !id(row.provenTxId)) || + (row.bytes !== null && + (typeof row.txid !== 'string' || Array.from(row.txid).length > 64 || Buffer.byteLength(row.txid) !== row.bytes)) + ) + throw new WERR_INVALID_OPERATION('Invalid global source identity') + const request = + row.txid === null + ? undefined + : await trx('proven_tx_reqs').where('txid', row.txid).first('provenTxReqId', 'provenTxId') + if ( + request !== undefined && + (!id(request.provenTxReqId) || (request.provenTxId !== null && !id(request.provenTxId))) + ) + throw new WERR_INVALID_OPERATION('Invalid global request identity') + const edges: Array<{ requestId: number; tableId: number; rowId: number }> = [] + if (row.provenTxId !== null) edges.push({ requestId: 0, tableId: 1, rowId: row.provenTxId }) + if (request !== undefined) { + edges.push({ requestId: request.provenTxReqId, tableId: 0, rowId: request.provenTxReqId }) + if (request.provenTxId !== null) + edges.push({ requestId: request.provenTxReqId, tableId: 1, rowId: request.provenTxId }) + } + await runInSeries(edges, async edge => { + if (edge.tableId === 1) { + const present = (await trx('proven_txs').where('provenTxId', edge.rowId).first('provenTxId')) !== undefined + await trx(names.guards).insert({ proofId: edge.rowId, present }).onConflict('proofId').merge({ present }) + } + await trx(names.edges) + .insert({ ...edge, transactionId: row.transactionId, userId: row.userId }) + .onConflict(['transactionId', 'requestId', 'tableId', 'rowId']) + .ignore() + }) + }) + return { + afterId: rows.at(-1)?.transactionId ?? state.afterId, + complete: rows.length < 256 ? 1 : 0 + } +} +async function copyStream(trx: Knex, state: Position): Promise> { + if (state.stream < profiles.length) return await addProfile(trx, state.stream, state) + if (state.stream < 10) return await addRelations(trx, state.stream, state) + if (state.stream === 10) return await addFields(trx, state) + return await addGlobal(trx, state) +} + +/** Exactly one bounded source page and its cursor commit together. */ +export async function copyGenerationPage( + k: Knex, + plan: Plan +): Promise<{ + complete: boolean + stream: number + copiedThrough: Position | undefined +}> { + return await k.transaction(async trx => { + await trx(metadata) + .where('id', 0) + .update({ complete: trx.ref('complete') }) + const states: Position[] = await trx(progress).select('*').orderBy('stream').limit(13) + if (states.length !== 12 || states.some((state, index) => state.stream !== index || !valid(state))) + throw new WERR_INVALID_OPERATION('Invalid generation progress') + const state = states.find(state => state.complete === 0) + if (state === undefined) { + await validateInstalled(trx, plan) + await trx(metadata).where('id', 0).update({ complete: true }) + return { complete: true, stream: 12, copiedThrough: undefined } + } + const next = await copyStream(trx, state) + await trx(progress).where('stream', state.stream).update(next) + return { complete: false, stream: state.stream, copiedThrough: { ...state, ...next } } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexGeneration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexGeneration.ts new file mode 100644 index 000000000..31f550766 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexGeneration.ts @@ -0,0 +1,287 @@ +import { runInSeries } from '../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { validateLegacy, legacySchema, retiredTables } from './snapshotSqliteLegacyOwnership' +import type { Knex } from 'knex' +import { readIdentity, identityDDL, type IdentityDefinition } from './snapshotSqliteIdentity' +import { numeric, relations, membershipTriggers, type MembershipNames } from './snapshotSqliteMembership' +import { validateSqliteSource } from './snapshotGlobalIndexSqlite' +import { snapshotGlobalIndexTriggers } from './snapshotGlobalIndexTriggers' +export const names: MembershipNames = { + profile: 'snapshot_profile_keys_v2', + relation: 'snapshot_relation_keys_v2', + certificate: 'snapshot_certificate_field_keys_v2', + edges: 'snapshot_global_edges_v2', + keys: 'snapshot_global_keys_v2', + guards: 'snapshot_global_guards_v2' +} +export const progress = 'snapshot_index_rebuild_v2' +export const metadata = 'snapshot_index_generation_v2' +export const oldProgress = [ + 'snapshot_profile_index_progress', + 'snapshot_relation_index_progress', + 'snapshot_certificate_index_progress', + 'snapshot_global_index_progress' +] +const q = (name: string) => '"' + name.replaceAll('"', '""') + '"' +interface Part { + name: string | null + coll: string + key: number + desc: number +} +export interface Plan { + identities: IdentityDefinition[] + fieldCollation: string + source: string + ddl: string[] + triggers: string[] +} +async function compositeOrder(k: Knex, table: string, columns: string[]): Promise { + const indexes: Array<{ + name: string + unique: number + partial: number + }> = await k.raw('PRAGMA index_list(??)', [table]) + const unique = indexes.filter(index => index.unique !== 0) + if (unique.length !== 1 || unique[0].partial !== 0) + throw new WERR_INVALID_OPERATION('Unsupported composite identity constraints') + const parts: Part[] = (await k.raw('PRAGMA index_xinfo(??)', [unique[0].name])).filter((part: Part) => part.key !== 0) + if ( + parts.length !== columns.length || + parts.some( + (part, i) => part.name !== columns[i] || part.desc !== 0 || !['BINARY', 'NOCASE', 'RTRIM'].includes(part.coll) + ) || + parts.slice(table === 'certificate_fields' ? 1 : 0).some(part => part.coll !== 'BINARY') + ) + throw new WERR_INVALID_OPERATION('Unsupported composite identity comparison') + const plan: Array<{ + detail: string + }> = await k.raw( + `EXPLAIN QUERY PLAN SELECT ${columns.map(q).join(',')} FROM ${q(table)} INDEXED BY ${q(unique[0].name)} ORDER BY ${columns.map(q).join(',')} LIMIT 1` + ) + if (plan.some(step => step.detail.includes('TEMP B-TREE'))) + throw new WERR_INVALID_OPERATION('Composite source order mismatch') + return parts[0].coll +} +function tables(fieldCollation: string): string[] { + const definitions: Array<[string, string, string[]]> = [ + [ + names.profile, + 'snapshotTableId INTEGER NOT NULL,snapshotUserId INTEGER NOT NULL,snapshotRowId INTEGER NOT NULL', + ['snapshotTableId', 'snapshotUserId', 'snapshotRowId'] + ], + [ + names.relation, + 'snapshotTableId INTEGER NOT NULL,snapshotUserId INTEGER NOT NULL,snapshotLeftId INTEGER NOT NULL,snapshotRightId INTEGER NOT NULL,snapshotMembership INTEGER NOT NULL', + ['snapshotTableId', 'snapshotUserId', 'snapshotLeftId', 'snapshotRightId'] + ], + [ + names.certificate, + `snapshotUserId INTEGER NOT NULL,snapshotFieldName VARCHAR(100) COLLATE ${fieldCollation} NOT NULL,snapshotCertificateId INTEGER NOT NULL,snapshotMembership INTEGER NOT NULL`, + ['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId'] + ], + [ + names.edges, + 'transactionId INTEGER NOT NULL,requestId INTEGER NOT NULL,tableId INTEGER NOT NULL,rowId INTEGER NOT NULL,userId INTEGER NOT NULL', + ['transactionId', 'requestId', 'tableId', 'rowId'] + ], + [ + names.keys, + 'tableId INTEGER NOT NULL,userId INTEGER NOT NULL,rowId INTEGER NOT NULL,refs BIGINT NOT NULL,present BOOLEAN NOT NULL', + ['tableId', 'userId', 'rowId'] + ], + [names.guards, 'proofId INTEGER NOT NULL,present BOOLEAN NOT NULL', ['proofId']], + [ + progress, + `stream INTEGER NOT NULL,afterId INTEGER NOT NULL,afterSecond INTEGER NOT NULL,afterText VARCHAR(100) COLLATE ${fieldCollation} NOT NULL,complete BOOLEAN NOT NULL`, + ['stream'] + ], + [ + metadata, + 'id INTEGER NOT NULL,source TEXT NOT NULL,complete BOOLEAN NOT NULL,legacy TEXT NOT NULL,retireTable INTEGER NOT NULL', + ['id'] + ] + ] + const indexes: Array<[string, string, string[]]> = [ + [ + 'snapshot_relation_right_v2', + names.relation, + ['snapshotTableId', 'snapshotUserId', 'snapshotRightId', 'snapshotLeftId'] + ], + [ + 'snapshot_relation_map_v2', + names.relation, + ['snapshotTableId', 'snapshotLeftId', 'snapshotRightId', 'snapshotUserId'] + ], + [ + 'snapshot_certificate_parent_v2', + names.certificate, + ['snapshotCertificateId', 'snapshotUserId', 'snapshotFieldName'] + ], + [ + 'snapshot_certificate_lookup_v2', + names.certificate, + ['snapshotFieldName', 'snapshotCertificateId', 'snapshotUserId'] + ], + ['snapshot_global_page_v2', names.keys, ['tableId', 'userId', 'present', 'rowId']], + ['snapshot_global_target_v2', names.keys, ['tableId', 'rowId', 'userId']], + ['snapshot_global_request_v2', names.edges, ['requestId', 'transactionId']] + ] + return [ + ...definitions.map( + ([name, columns, primary]) => `CREATE TABLE ${q(name)} (${columns},PRIMARY KEY (${primary.map(q).join(',')}))` + ), + ...indexes.map(([name, table, columns]) => `CREATE INDEX ${q(name)} ON ${q(table)} (${columns.map(q).join(',')})`) + ] +} +export async function readPlan(k: Knex): Promise { + if (!String(k.client.config.client).includes('sqlite')) + throw new WERR_INVALID_OPERATION('SQLite rebuild requires SQLite') + await validateSqliteSource(k) + const identities: IdentityDefinition[] = [] + await runInSeries(numeric, async source => { + identities.push(await readIdentity(k, source)) + }) + await runInSeries(relations, async relation => { + await compositeOrder(k, relation.table, [relation.leftKey, relation.rightKey]) + }) + const fieldCollation = await compositeOrder(k, 'certificate_fields', ['fieldName', 'certificateId']) + const sources = [ + ...numeric.map(source => source.table), + ...relations.map(relation => relation.table), + 'certificate_fields' + ] + const schema = await k('sqlite_master') + .whereIn('tbl_name', sources) + .whereIn('type', ['table', 'index']) + .select('type', 'name', 'tbl_name', 'sql') + .orderBy(['type', 'name']) + const source = JSON.stringify(schema) + const ddl = [...tables(fieldCollation), ...identities.flatMap(identityDDL)] + const observer = snapshotGlobalIndexTriggers(false) + .filter(trigger => trigger.table === 'snapshot_global_edges') + .map(trigger => + trigger.sql + .replaceAll('snapshot_global_edges', names.edges) + .replaceAll('snapshot_global_keys', names.keys) + .replaceAll('snapshot_global_guards', names.guards) + .replaceAll('snapshot_global_edge_', 'snapshot_global_v2_edge_') + ) + return { + identities, + fieldCollation, + source, + ddl, + triggers: [...observer, ...membershipTriggers(identities, names)] + } +} +async function validateInstallLock(k: Knex): Promise { + const lock = await k('sqlite_master').where({ type: 'table', name: 'snapshot_index_install_lock_v2' }).first('sql') + if (lock?.sql !== 'CREATE TABLE snapshot_index_install_lock_v2(id INTEGER PRIMARY KEY)') + throw new WERR_INVALID_OPERATION('Invalid rebuild lock definition') +} + +/** Exact generated DDL ownership; never adopt a similarly named foreign object. */ +export async function validateInstalled(k: Knex, plan: Plan): Promise { + await validateInstallLock(k) + if ((await readPlan(k)).source !== plan.source) throw new WERR_INVALID_OPERATION('Rebuild source schema changed') + const expected = [...plan.ddl, ...plan.triggers] + await runInSeries(expected, async sql => { + const match = /^CREATE (TABLE|INDEX|TRIGGER) ("[^"]+"|\w+)/.exec(sql) + if (!match) throw new WERR_INVALID_OPERATION('Invalid generated schema definition') + const name = match[2].replaceAll('"', '') + const rows: Array<{ + sql: string + }> = await k('sqlite_master').where({ type: match[1].toLowerCase(), name }).select('sql') + if (rows.length !== 1 || rows[0].sql !== sql) + throw new WERR_INVALID_OPERATION('Rebuild schema definition mismatch: ' + name) + }) + const rows: Array<{ + id: number + source: string + complete: number + legacy: string + retireTable: number + }> = await k(metadata).select('*').limit(2) + if ( + rows.length !== 1 || + rows[0].id !== 0 || + rows[0].source !== plan.source || + ![0, 1].includes(rows[0].complete) || + !Number.isSafeInteger(rows[0].retireTable) || + rows[0].retireTable < 0 || + rows[0].retireTable > retiredTables.length || + typeof rows[0].legacy !== 'string' + ) + throw new WERR_INVALID_OPERATION('Rebuild source binding mismatch') +} +/** Atomically install owned observers and invalidate the previous generation. */ +export async function installGeneration(k: Knex, config?: Knex.MigratorConfig): Promise { + return await k.transaction(async trx => { + // A harmless DDL write reserves SQLite's writer lock before schema observation. + // The connection remains in this transaction until all observer swaps commit. + await trx.raw('CREATE TABLE IF NOT EXISTS snapshot_index_install_lock_v2(id INTEGER PRIMARY KEY)') + await validateInstallLock(trx) + await trx.raw('UPDATE snapshot_index_install_lock_v2 SET id=id') + const plan = await readPlan(trx) + if (await trx.schema.hasTable(metadata)) { + await validateInstalled(trx, plan) + return plan + } + const legacyNames = await validateLegacy(trx, config) + const namesToOwn = plan.ddl + .map(sql => /^CREATE (?:TABLE|INDEX) "([^"]+)"/.exec(sql)?.[1]) + .filter((name): name is string => name !== undefined) + if ((await trx('sqlite_master').whereIn('name', namesToOwn)).length !== 0) + throw new WERR_INVALID_OPERATION('Orphan rebuild schema refuses adoption') + await runInSeries(plan.ddl, async sql => { + await trx.raw(sql) + }) + const old: Array<{ + name: string + }> = await trx('sqlite_master') + .where('type', 'trigger') + .whereIn('tbl_name', [ + ...numeric.map(source => source.table), + ...relations.map(relation => relation.table), + 'certificate_fields' + ]) + .select('name') + await runInSeries(old, async trigger => { + if (/^snapshot_(profile|relation|certificate|global)_/.test(trigger.name)) { + if (!legacyNames.has(trigger.name)) throw new WERR_INVALID_OPERATION('Unknown legacy source trigger') + await trx.raw('DROP TRIGGER ??', [trigger.name]) + } + }) + await runInSeries( + snapshotGlobalIndexTriggers(false).filter(trigger => trigger.table === 'snapshot_global_edges'), + async trigger => { + await trx.raw('DROP TRIGGER ??', [trigger.name]) + } + ) + const legacy = JSON.stringify(await legacySchema(trx)) + await runInSeries(plan.triggers, async sql => { + await trx.raw(sql) + }) + await runInSeries(oldProgress, async table => { + await trx(table).update({ complete: false }) + }) + await trx(metadata).insert({ + id: 0, + source: plan.source, + complete: false, + legacy, + retireTable: 0 + }) + await trx(progress).insert( + Array.from({ length: 12 }, (_, stream) => ({ + stream, + afterId: 0, + afterSecond: 0, + afterText: '', + complete: false + })) + ) + return plan + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexMigration.ts new file mode 100644 index 000000000..4fed0edea --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexMigration.ts @@ -0,0 +1,56 @@ +import { runInSeries } from '../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { Knex } from 'knex' +import { installGeneration, metadata, readPlan, validateInstalled } from './snapshotSqliteIndexGeneration' +import { copyGenerationPage } from './snapshotSqliteIndexBootstrap' +import { retireGenerationPage } from './snapshotSqliteIndexRetirement' +/** Pull one page at a time, yielding foreground execution between commits. */ +async function completePages(read: () => Promise<{ complete: boolean }>): Promise { + let complete = false + function* pending() { + while (!complete) yield undefined + } + await runInSeries(pending(), async () => { + complete = (await read()).complete + if (!complete) await new Promise(resolve => setTimeout(resolve, 0)) + }) +} + +/** Additive SQLite repair; MySQL retains its existing native maintenance. */ +export async function migrateGeneration(k: Knex): Promise { + if (String(k.client.config.client).includes('mysql')) return + if (k.isTransaction) throw new WERR_INVALID_OPERATION('SQLite generation requires independent bounded transactions') + const plan = await installGeneration(k, k.client.config.migrations) + await completePages(() => copyGenerationPage(k, plan)) + await completePages(() => retireGenerationPage(k, plan)) +} +/** Source rows are unchanged; an older snapshot implementation cannot use retired indexes. */ +export function refuseGenerationDowngrade(k: Knex): Promise { + if (String(k.client.config.client).includes('mysql')) return Promise.resolve() + return Promise.reject( + new WERR_INVALID_OPERATION( + 'SQLite snapshot generation downgrade is unsupported; preserve source rows and use a forward migration' + ) + ) +} +/** Only StorageKnex.dropAllData may remove this generation without rebuilding legacy indexes. */ +export async function dropGenerationForDataDeletion(k: Knex): Promise { + if (!String(k.client.config.client).includes('sqlite')) return + await k.transaction(async trx => { + if (!(await trx.schema.hasTable(metadata))) return + await trx(metadata) + .where('id', 0) + .update({ retireTable: trx.ref('retireTable') }) + const plan = await readPlan(trx) + await validateInstalled(trx, plan) + await runInSeries([...plan.triggers].reverse(), async sql => { + const name = /^CREATE TRIGGER ("[^"]+"|\w+)/.exec(sql)![1].replaceAll('"', '') + await trx.raw('DROP TRIGGER ??', [name]) + }) + await runInSeries([...plan.ddl].reverse(), async sql => { + const match = /^CREATE TABLE "([^"]+)"/.exec(sql) + if (match) await trx.schema.dropTable(match[1]) + }) + await trx.schema.dropTable('snapshot_index_install_lock_v2') + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexRetirement.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexRetirement.ts new file mode 100644 index 000000000..28cf2ae11 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexRetirement.ts @@ -0,0 +1,59 @@ +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { Knex } from 'knex' +import { metadata, validateInstalled, type Plan } from './snapshotSqliteIndexGeneration' +import { legacySchema, retiredTables, type SchemaObject } from './snapshotSqliteLegacyOwnership' + +/** Remove at most 256 obsolete rows; drop only an already-empty owned table. */ +export async function retireGenerationPage( + k: Knex, + plan: Plan +): Promise<{ complete: boolean; removed: number; table: string | undefined }> { + return await k.transaction(async trx => { + await trx(metadata) + .where('id', 0) + .update({ retireTable: trx.ref('retireTable') }) + await validateInstalled(trx, plan) + const state: { complete: number; legacy: string; retireTable: number } = await trx(metadata).where('id', 0).first() + if (state.complete !== 1) throw new WERR_INVALID_OPERATION('Cannot retire before generation copy completes') + let expected: SchemaObject[] + try { + expected = JSON.parse(state.legacy) + } catch { + throw new WERR_INVALID_OPERATION('Invalid legacy ownership receipt') + } + if ( + !Array.isArray(expected) || + expected.some( + row => + typeof row !== 'object' || + row === null || + !retiredTables.includes(row.tbl_name) || + !['table', 'index'].includes(row.type) || + typeof row.name !== 'string' || + (row.sql !== null && typeof row.sql !== 'string') + ) + ) + throw new WERR_INVALID_OPERATION('Invalid legacy ownership receipt') + const remaining = expected.filter(row => retiredTables.indexOf(row.tbl_name) >= state.retireTable) + if (JSON.stringify(await legacySchema(trx)) !== JSON.stringify(remaining)) + throw new WERR_INVALID_OPERATION('Legacy retirement schema changed') + const table = retiredTables[state.retireTable] + if (table === undefined) return { complete: true, removed: 0, table } + const removed = await trx(table) + .whereIn('_rowid_', trx(table).select('_rowid_').orderBy('_rowid_').limit(256)) + .delete() + if (!Number.isSafeInteger(removed) || removed < 0 || removed > 256) + throw new WERR_INVALID_OPERATION('Invalid retirement row count') + if (removed === 0) { + await trx.schema.dropTable(table) + await trx(metadata) + .where('id', 0) + .update({ retireTable: state.retireTable + 1 }) + } + return { + complete: removed === 0 && state.retireTable + 1 === retiredTables.length, + removed, + table + } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexState.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexState.ts new file mode 100644 index 000000000..a3c879337 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteIndexState.ts @@ -0,0 +1,37 @@ +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { Knex } from 'knex' +import { readPlan, validateInstalled, metadata, progress } from './snapshotSqliteIndexGeneration' +import { valid, type Position } from './snapshotSqliteIndexBootstrap' +export const migration = '2026-10-02-001 repair snapshot SQLite conflict maintenance' +export type SnapshotIndexState = boolean | 'v2' +/** The caller holds the same pinned view that will read all selected pages. */ +export async function readGenerationIndexState( + k: Knex, + config?: Knex.MigratorConfig +): Promise { + if (!String(k.client.config.client).includes('sqlite') || !(await k.schema.hasTable(metadata))) return undefined + const plan = await readPlan(k) + await validateInstalled(k, plan) + const rows: Position[] = await k(progress).select('*').orderBy('stream').limit(13) + if (rows.length !== 12 || rows.some((row, index) => row.stream !== index || !valid(row))) + throw new WERR_INVALID_OPERATION('Invalid generation progress') + const state = await k(metadata).where('id', 0).first('complete') + const journal = config?.tableName ?? 'knex_migrations' + // SQLite's Knex hasTable query ignores withSchema; inspect the selected + // database catalog explicitly before reading its migration journal. + const catalog = k('sqlite_master').where({ type: 'table', name: journal }).first('name') + if (config?.schemaName !== undefined) void catalog.withSchema(config.schemaName) + let published = false + if ((await catalog) !== undefined) { + const query = k(journal).where('name', migration) + if (config?.schemaName !== undefined) void query.withSchema(config.schemaName) + published = (await query.first('name')) !== undefined + } + if (state.complete === 0) { + if (published) throw new WERR_INVALID_OPERATION('Published generation is incomplete') + return false + } + if (rows.some(row => row.complete !== 1)) + throw new WERR_INVALID_OPERATION('Completed generation has unfinished streams') + return published ? 'v2' : false +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteLegacyOwnership.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteLegacyOwnership.ts new file mode 100644 index 000000000..40b7322dc --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteLegacyOwnership.ts @@ -0,0 +1,78 @@ +import { runInSeries } from '../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { Knex } from 'knex' +import { snapshotGlobalIndexTriggers } from './snapshotGlobalIndexTriggers' +import { addSnapshotProfileIndexes, readSnapshotProfileIndexState } from './snapshotProfileIndexMigration' +import { addSnapshotRelationIndexes, readSnapshotRelationIndexState } from './snapshotRelationIndexMigration' +import { addSnapshotCertificateIndexes, readSnapshotCertificateIndexState } from './snapshotCertificateIndexMigration' +import { addSnapshotGlobalIndexes, readSnapshotGlobalIndexState } from './snapshotGlobalIndexMigration' +import { legacyNames } from './snapshotSqliteMembership' +export const retiredTables = [ + legacyNames.edges, + legacyNames.keys, + legacyNames.guards, + legacyNames.profile, + legacyNames.relation, + legacyNames.certificate +] +export interface SchemaObject { + type: string + name: string + tbl_name: string + sql: string | null +} +export async function legacySchema(k: Knex): Promise { + return await k('sqlite_master') + .whereIn('tbl_name', retiredTables) + .select('type', 'name', 'tbl_name', 'sql') + .orderBy(['type', 'name']) +} +function legacyTriggerNames(): Set { + const names = new Set(snapshotGlobalIndexTriggers(false).map(trigger => trigger.name)) + for (let id = 0; id < 8; id++) + for (const event of ['insert', 'update', 'delete']) names.add(`snapshot_profile_${id}_${event}`) + for (let id = 0; id < 2; id++) + for (const side of ['map', 'left', 'right']) + for (const event of ['insert', 'delete', 'before_update', 'after_update']) + names.add(`snapshot_relation_${id}_${side}_${event}`) + for (const side of ['field', 'parent']) + for (const event of ['insert', 'delete', 'before_update', 'after_update']) + names.add(`snapshot_certificate_${side}_${event}`) + return names +} + +/** Existing validators own the old definitions; complete states prevent bootstrap scans. */ +export async function validateLegacy(k: Knex, config?: Knex.MigratorConfig): Promise> { + await runInSeries( + [ + readSnapshotProfileIndexState, + readSnapshotRelationIndexState, + readSnapshotCertificateIndexState, + readSnapshotGlobalIndexState + ], + async read => { + if (!(await read(k, config))) throw new WERR_INVALID_OPERATION('Prior snapshot migrations must be complete') + } + ) + await runInSeries( + [addSnapshotProfileIndexes, addSnapshotRelationIndexes, addSnapshotCertificateIndexes, addSnapshotGlobalIndexes], + async install => { + await install(k) + } + ) + const names = legacyTriggerNames() + const foreign: Array<{ + type: string + name: string + tbl_name: string + sql: string + }> = await k('sqlite_master').whereIn('type', ['trigger', 'view']).select('type', 'name', 'tbl_name', 'sql') + for (const object of foreign) + if ( + !(object.type === 'trigger' && names.has(object.name)) && + (retiredTables.includes(object.tbl_name) || + retiredTables.some(table => new RegExp(String.raw`\b${table}\b`, 'i').test(object.sql))) + ) + throw new WERR_INVALID_OPERATION('Unowned object references legacy auxiliary data') + return names +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteMembership.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteMembership.ts new file mode 100644 index 000000000..5f6757627 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSqliteMembership.ts @@ -0,0 +1,249 @@ +import { + observeIdentity, + displacedIdentity, + finishIdentity, + type IdentityDefinition, + type SourceIdentity +} from './snapshotSqliteIdentity' + +// Pure maintenance SQL shared by generation installation and focused fixture tests. +export const profiles = [ + 'transactions', + 'outputs', + 'certificates', + 'tx_labels', + 'output_baskets', + 'output_tags', + 'commissions', + 'sync_states' +] +export const numeric: SourceIdentity[] = [ + { table: 'transactions', key: 'transactionId', owner: 'userId' }, + { table: 'outputs', key: 'outputId', owner: 'userId' }, + { table: 'certificates', key: 'certificateId', owner: 'userId' }, + { table: 'tx_labels', key: 'txLabelId', owner: 'userId' }, + { table: 'output_baskets', key: 'basketId', owner: 'userId' }, + { table: 'output_tags', key: 'outputTagId', owner: 'userId' }, + { table: 'commissions', key: 'commissionId', owner: 'userId' }, + { table: 'sync_states', key: 'syncStateId', owner: 'userId' }, + { table: 'proven_txs', key: 'provenTxId' }, + { table: 'proven_tx_reqs', key: 'provenTxReqId' } +] +export const relations = [ + { + table: 'tx_labels_map', + left: 'tx_labels', + leftKey: 'txLabelId', + right: 'transactions', + rightKey: 'transactionId' + }, + { + table: 'output_tags_map', + left: 'output_tags', + leftKey: 'outputTagId', + right: 'outputs', + rightKey: 'outputId' + } +] +type Event = 'INSERT' | 'UPDATE' | 'DELETE' +const q = (name: string): string => '"' + name.replaceAll('"', '""') + '"' +const relationColumns = 'snapshotTableId,snapshotUserId,snapshotLeftId,snapshotRightId' +const fieldColumns = 'snapshotUserId,snapshotFieldName,snapshotCertificateId' + +export interface MembershipNames { + profile: string + relation: string + certificate: string + edges: string + keys: string + guards: string +} +export const legacyNames: MembershipNames = { + profile: 'snapshot_profile_keys', + relation: 'snapshot_relation_keys', + certificate: 'snapshot_certificate_field_keys', + edges: 'snapshot_global_edges', + keys: 'snapshot_global_keys', + guards: 'snapshot_global_guards' +} +export function membershipBodies(names: MembershipNames = legacyNames) { + function relationInsert(select: string, bit: number): string { + return `INSERT INTO ${names.relation}(${relationColumns},snapshotMembership) ${select} ON CONFLICT(${relationColumns}) DO UPDATE SET snapshotMembership=snapshotMembership|${bit}; ` + } + function fieldInsert(select: string, bit: number): string { + return `INSERT INTO ${names.certificate}(${fieldColumns},snapshotMembership) ${select} ON CONFLICT(${fieldColumns}) DO UPDATE SET snapshotMembership=snapshotMembership|${bit}; ` + } + function edgeInsert(select: string): string { + return `INSERT INTO ${names.edges}(transactionId,requestId,tableId,rowId,userId) ${select} ON CONFLICT(transactionId,requestId,tableId,rowId) DO NOTHING; ` + } + function ensureProof(expression: string): string { + return `INSERT INTO ${names.guards}(proofId,present) SELECT ${expression},EXISTS(SELECT 1 FROM proven_txs WHERE provenTxId=${expression}) WHERE ${expression} IS NOT NULL ON CONFLICT(proofId) DO UPDATE SET present=excluded.present; ` + } + + function profileMembership(table: string, key: string, context: string, owners: string): string { + let sql = '' + const profileId = profiles.indexOf(table) + if (profileId !== -1) { + sql += `DELETE FROM ${names.profile} WHERE snapshotTableId=${profileId} AND (snapshotUserId,snapshotRowId) IN (${owners}); ` + sql += `INSERT INTO ${names.profile}(snapshotTableId,snapshotUserId,snapshotRowId) SELECT ${profileId},userId,${q(key)} FROM ${q(table)} WHERE ${q(key)}=${context}.${q(key)} ON CONFLICT(snapshotTableId,snapshotUserId,snapshotRowId) DO NOTHING; ` + } + return sql + } + function parentMembership(table: string, key: string, context: string, owners: string): string { + let sql = '' + for (const [id, relation] of relations.entries()) + for (const side of ['left', 'right'] as const) { + if (relation[side] !== table) continue + const bit = side === 'left' ? 1 : 2 + const index = side === 'left' ? 'snapshotLeftId' : 'snapshotRightId' + const where = `snapshotTableId=${id} AND (snapshotUserId,${index}) IN (${owners})` + sql += `UPDATE ${names.relation} SET snapshotMembership=snapshotMembership&${3 ^ bit} WHERE ${where}; DELETE FROM ${names.relation} WHERE ${where} AND snapshotMembership=0; ` + sql += relationInsert( + `SELECT ${id},p.userId,m.${relation.leftKey},m.${relation.rightKey},${bit} FROM ${q(table)} p JOIN ${relation.table} m ON m.${key}=p.${key} WHERE p.${key}=${context}.${key}`, + bit + ) + } + if (table === 'certificates') { + const where = `(snapshotUserId,snapshotCertificateId) IN (${owners})` + sql += `UPDATE ${names.certificate} SET snapshotMembership=snapshotMembership&1 WHERE ${where}; DELETE FROM ${names.certificate} WHERE ${where} AND snapshotMembership=0; ` + sql += fieldInsert( + `SELECT c.userId,f.fieldName,f.certificateId,2 FROM certificates c JOIN ${names.certificate} k ON k.snapshotCertificateId=c.certificateId AND (k.snapshotMembership&1)=1 JOIN certificate_fields f ON f.fieldName=k.snapshotFieldName AND f.certificateId=k.snapshotCertificateId WHERE c.certificateId=${context}.certificateId`, + 2 + ) + } + return sql + } + function globalMembership(table: string, context: string, ids: string): string { + let sql = '' + if (table === 'transactions') { + sql += `DELETE FROM ${names.edges} WHERE transactionId IN (${ids}); ` + sql += ensureProof(`(SELECT provenTxId FROM transactions WHERE transactionId=${context}.transactionId)`) + sql += ensureProof( + `(SELECT r.provenTxId FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid WHERE t.transactionId=${context}.transactionId)` + ) + sql += edgeInsert( + `SELECT t.transactionId,0,1,t.provenTxId,t.userId FROM transactions t WHERE t.transactionId=${context}.transactionId AND t.provenTxId IS NOT NULL` + ) + sql += edgeInsert( + `SELECT t.transactionId,r.provenTxReqId,0,r.provenTxReqId,t.userId FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid WHERE t.transactionId=${context}.transactionId` + ) + sql += edgeInsert( + `SELECT t.transactionId,r.provenTxReqId,1,r.provenTxId,t.userId FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid WHERE t.transactionId=${context}.transactionId AND r.provenTxId IS NOT NULL` + ) + } + if (table === 'proven_tx_reqs') { + sql += `DELETE FROM ${names.edges} WHERE requestId IN (${ids}); ` + const proof = `(SELECT provenTxId FROM proven_tx_reqs WHERE provenTxReqId=${context}.provenTxReqId)` + sql += ensureProof(proof) + sql += edgeInsert( + `SELECT t.transactionId,r.provenTxReqId,0,r.provenTxReqId,t.userId FROM proven_tx_reqs r JOIN transactions t ON t.txid=r.txid WHERE r.provenTxReqId=${context}.provenTxReqId` + ) + sql += edgeInsert( + `SELECT t.transactionId,r.provenTxReqId,1,r.provenTxId,t.userId FROM proven_tx_reqs r JOIN transactions t ON t.txid=r.txid WHERE r.provenTxReqId=${context}.provenTxReqId AND r.provenTxId IS NOT NULL` + ) + sql += `DELETE FROM ${names.guards} WHERE proofId=${proof} AND NOT EXISTS(SELECT 1 FROM ${names.keys} WHERE tableId=1 AND rowId=proofId); ` + } + if (table === 'proven_txs') { + sql += `UPDATE ${names.guards} SET present=0 WHERE proofId IN (${ids}); UPDATE ${names.keys} SET present=0 WHERE tableId=1 AND rowId IN (${ids}); DELETE FROM ${names.guards} WHERE proofId IN (${ids}) AND NOT EXISTS(SELECT 1 FROM ${names.keys} WHERE tableId=1 AND rowId=proofId); ` + sql += `UPDATE ${names.guards} SET present=1 WHERE proofId=${context}.provenTxId AND EXISTS(SELECT 1 FROM proven_txs WHERE provenTxId=${context}.provenTxId); UPDATE ${names.keys} SET present=1 WHERE tableId=1 AND rowId=${context}.provenTxId AND EXISTS(SELECT 1 FROM proven_txs WHERE provenTxId=${context}.provenTxId); ` + } + return sql + } + + function numericMembership(d: IdentityDefinition, event: Event): string { + const { table, key, owner } = d.source + const context = event === 'DELETE' ? 'OLD' : 'NEW' + const ownerColumn = owner ? ',OLD.' + q(owner) + ' AS ' + q(owner) : '' + const retired = + event === 'DELETE' ? `SELECT OLD.${q(key)} AS ${q(key)}${ownerColumn}` : displacedIdentity(d, event === 'UPDATE') + const ids = `SELECT ${q(key)} FROM (${retired})` + const owners = owner ? `SELECT ${q(owner)},${q(key)} FROM (${retired})` : '' + const sql = + profileMembership(table, key, context, owners) + + parentMembership(table, key, context, owners) + + globalMembership(table, context, ids) + return sql + finishIdentity(d, event) + } + + function compositeMembership(table: string, event: Event): string { + const context = event === 'DELETE' ? 'OLD' : 'NEW' + const contexts = event === 'UPDATE' ? ['OLD', 'NEW'] : [context] + if (table === 'certificate_fields') { + let sql = contexts + .map( + value => + `DELETE FROM ${names.certificate} WHERE snapshotFieldName=${value}.fieldName AND snapshotCertificateId=${value}.certificateId; ` + ) + .join('') + sql += fieldInsert( + `SELECT userId,fieldName,certificateId,1 FROM certificate_fields WHERE fieldName=${context}.fieldName AND certificateId=${context}.certificateId`, + 1 + ) + sql += fieldInsert( + `SELECT c.userId,f.fieldName,f.certificateId,2 FROM certificate_fields f JOIN certificates c ON c.certificateId=f.certificateId WHERE f.fieldName=${context}.fieldName AND f.certificateId=${context}.certificateId`, + 2 + ) + return sql + } + const id = relations.findIndex(relation => relation.table === table) + const relation = relations[id] + let sql = contexts + .map( + value => + `DELETE FROM ${names.relation} WHERE snapshotTableId=${id} AND snapshotLeftId=${value}.${relation.leftKey} AND snapshotRightId=${value}.${relation.rightKey}; ` + ) + .join('') + for (const side of ['left', 'right'] as const) { + const key = side === 'left' ? relation.leftKey : relation.rightKey + const bit = side === 'left' ? 1 : 2 + sql += relationInsert( + `SELECT ${id},p.userId,m.${relation.leftKey},m.${relation.rightKey},${bit} FROM ${table} m JOIN ${relation[side]} p ON p.${key}=m.${key} WHERE m.${relation.leftKey}=${context}.${relation.leftKey} AND m.${relation.rightKey}=${context}.${relation.rightKey}`, + bit + ) + } + return sql + } + + return { numeric: numericMembership, composite: compositeMembership } +} + +function changed(columns: string[]): string { + return [...new Set(columns)] + .map(column => `CAST(OLD.${q(column)} AS BLOB) IS NOT CAST(NEW.${q(column)} AS BLOB)`) + .join(' OR ') +} + +function numericTriggers(d: IdentityDefinition, bodies: ReturnType): string[] { + const sql: string[] = [] + const fields = d.columns.map(column => column.name) + if (d.source.table === 'transactions') fields.push('txid', 'provenTxId') + if (d.source.table === 'proven_tx_reqs') fields.push('provenTxId') + for (const event of ['INSERT', 'UPDATE', 'DELETE'] as const) { + const when = event === 'UPDATE' ? ' WHEN ' + changed(fields) : '' + if (event !== 'DELETE') + sql.push( + `CREATE TRIGGER ${q('snapshot_identity_before_' + d.source.table + '_' + event)} BEFORE ${event} ON ${q(d.source.table)}${when} BEGIN ${observeIdentity(d, event === 'UPDATE')} END` + ) + sql.push( + `CREATE TRIGGER ${q('snapshot_identity_after_' + d.source.table + '_' + event)} AFTER ${event} ON ${q(d.source.table)}${when} BEGIN ${bodies.numeric(d, event)} END` + ) + } + return sql +} + +export function membershipTriggers(definitions: IdentityDefinition[], names: MembershipNames = legacyNames): string[] { + const sql: string[] = [] + const bodies = membershipBodies(names) + for (const d of definitions) sql.push(...numericTriggers(d, bodies)) + for (const table of [...relations.map(relation => relation.table), 'certificate_fields']) { + const relation = relations.find(relation => relation.table === table) + const fields = relation ? [relation.leftKey, relation.rightKey] : ['fieldName', 'certificateId', 'userId'] + for (const event of ['INSERT', 'UPDATE', 'DELETE'] as const) { + const when = event === 'UPDATE' ? ' WHEN ' + changed(fields) : '' + sql.push( + `CREATE TRIGGER ${q('snapshot_identity_after_' + table + '_' + event)} AFTER ${event} ON ${q(table)}${when} BEGIN ${bodies.composite(table, event)} END` + ) + } + } + return sql +} diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSyncMigration.test.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSyncMigration.test.ts new file mode 100644 index 000000000..746efac2f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSyncMigration.test.ts @@ -0,0 +1,112 @@ +import { knex, type Knex } from 'knex' +import { addSnapshotSyncTables, removeSnapshotSyncTables } from './snapshotSyncMigration' + +const names = ['snapshot_sync_sessions', 'snapshot_sync_ids', 'snapshot_sync_primary_epochs'] + +test('SQLite migration preserves user data, counts primary transitions and removes only its auxiliary objects', async () => { + const k = knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) + try { + await k.raw('PRAGMA foreign_keys = ON') + await k.schema.createTable('users', table => { + table.increments('userId') + table.string('activeStorage') + }) + await k('users').insert({ userId: 1, activeStorage: 'source' }) + await addSnapshotSyncTables(k) + await addSnapshotSyncTables(k) + await k('users').where({ userId: 1 }).update({ userId: 1 }) + await k('users').where({ userId: 1 }).update({ activeStorage: 'source' }) + expect(await k('snapshot_sync_primary_epochs')).toHaveLength(0) + await k('users').where({ userId: 1 }).update({ activeStorage: 'destination' }) + await k('users').where({ userId: 1 }).update({ activeStorage: 'source' }) + expect(await k('snapshot_sync_primary_epochs')).toEqual([{ userId: 1, epoch: 2 }]) + await expect(k('snapshot_sync_primary_epochs').insert({ userId: 999, epoch: 1 })).rejects.toThrow('FOREIGN KEY') + const columns = await k('snapshot_sync_ids').columnInfo() + expect(Object.keys(columns)).toEqual(['userId', 'sourceStorageIdentityKey', 'entity', 'incomingId', 'localId']) + await removeSnapshotSyncTables(k) + await removeSnapshotSyncTables(k) + expect(await k('users')).toEqual([{ userId: 1, activeStorage: 'source' }]) + for (const name of names) expect(await k.schema.hasTable(name)).toBe(false) + } finally { + await k.destroy() + } +}) + +test('MySQL migration repairs separately committed DDL without duplicating tables, keys or its trigger', async () => { + // Compile every builder against real Knex MySQL SQL generation. The native + // MySQL fixture separately qualifies execution and trigger semantics. + const compiler = knex({ client: 'mysql2' }) + const tables = new Set() + const foreignKeys = new Set() + const ddl: string[] = [] + let trigger = false + let interrupted = false + const database = { + client: { config: { client: 'mysql2' } }, + schema: { + hasTable: async (name: string) => tables.has(name), + createTable: async (name: string, build: (table: Knex.CreateTableBuilder) => void) => { + ddl.push( + ...compiler.schema + .createTable(name, build) + .toSQL() + .map(query => query.sql) + ) + tables.add(name) + }, + table: async (name: string, build: (table: Knex.AlterTableBuilder) => void) => { + if (!interrupted) { + interrupted = true + throw new Error('interrupted after table DDL') + } + ddl.push( + ...compiler.schema + .table(name, build) + .toSQL() + .map(query => query.sql) + ) + foreignKeys.add(name + '_user') + } + }, + raw: async (sql: string, bindings?: string[]) => { + if (sql.includes('TABLE_CONSTRAINTS')) { + expect(bindings).toEqual([expect.stringMatching(/^snapshot_sync_/), expect.stringMatching(/_user$/)]) + return [foreignKeys.has(bindings![1]) ? [{ name: bindings![1] }] : []] + } + if (sql.includes('information_schema.TRIGGERS')) { + expect(bindings).toEqual(['snapshot_sync_primary_change']) + return [trigger ? [{ name: 'snapshot_sync_primary_change' }] : []] + } + if (sql.startsWith('CREATE TRIGGER snapshot_sync_primary_change')) { + ddl.push(sql) + trigger = true + return [] + } + throw new Error('Unexpected migration query') + } + } as unknown as Knex + try { + await expect(addSnapshotSyncTables(database)).rejects.toThrow('interrupted after table DDL') + expect([...tables]).toEqual(names) + expect(foreignKeys.size).toBe(0) + await addSnapshotSyncTables(database) + const completed = [...ddl] + await addSnapshotSyncTables(database) + expect(ddl).toEqual(completed) + expect(foreignKeys.size).toBe(3) + expect(trigger).toBe(true) + for (const sql of ddl.filter(sql => sql.startsWith('create table'))) + expect(sql).toContain('`userId` int unsigned not null') + const mappings = ddl.find(sql => sql.startsWith('create table `snapshot_sync_ids`'))! + expect(mappings).toContain('`incomingId` bigint unsigned not null') + expect(mappings).toContain('`localId` bigint unsigned not null') + expect(mappings).toContain('primary key (`userId`, `sourceStorageIdentityKey`, `entity`, `incomingId`)') + expect(ddl.filter(sql => sql.startsWith('alter table'))).toHaveLength(3) + for (const sql of ddl.filter(sql => sql.startsWith('alter table'))) + expect(sql).toContain('foreign key (`userId`) references `users` (`userId`)') + expect(ddl.at(-1)).toContain('NOT (OLD.activeStorage <=> NEW.activeStorage)') + expect(ddl.at(-1)).toContain('ON DUPLICATE KEY UPDATE epoch = epoch + 1') + } finally { + await compiler.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSyncMigration.ts b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSyncMigration.ts new file mode 100644 index 000000000..0a0ded97b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/schema/snapshotSyncMigration.ts @@ -0,0 +1,93 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../utility/runInSeries' + +export const SNAPSHOT_SYNC_MIGRATION = '2026-09-30-001 add durable snapshot sync' + +/** Auxiliary tables only: preserve standard-table traversal and legacy sync-map JSON. */ +export async function addSnapshotSyncTables(knex: Knex): Promise { + const mysql = String(knex.client.config.client).includes('mysql') + const userColumn = (table: Knex.CreateTableBuilder): Knex.ColumnBuilder => { + const column = table.integer('userId').unsigned().notNullable() + // MySQL creates foreign keys in separate DDL. Install/recover them below. + if (!mysql) column.references('userId').inTable('users') + return column + } + // MySQL DDL commits implicitly. Creations are idempotent after interruption. + if (!(await knex.schema.hasTable('snapshot_sync_sessions'))) { + await knex.schema.createTable('snapshot_sync_sessions', table => { + userColumn(table) + table.string('sourceStorageIdentityKey', 130).notNullable() + table.string('sessionId', 64).notNullable() + table.string('snapshotId', 64).notNullable() + table.integer('version').notNullable() + table.bigInteger('sourceUserId').unsigned().notNullable() + table.string('identityKey', 130).notNullable() + table.string('destinationStorageIdentityKey', 130).notNullable() + table.string('chain', 8).notNullable() + table.string('activeStorage', 130).nullable() + table.string('sourceActiveStorage', 130).nullable() + table.string('sourceUserCreatedAt', 64).notNullable() + table.string('sourceUserUpdatedAt', 64).notNullable() + table.bigInteger('primaryEpoch').notNullable() + table.bigInteger('expiresAt').notNullable() + table.integer('tableIndex').notNullable() + table.integer('sequence').notNullable() + table.text('cursor').nullable() + table.primary(['userId', 'sourceStorageIdentityKey']) + }) + } + if (!(await knex.schema.hasTable('snapshot_sync_ids'))) { + await knex.schema.createTable('snapshot_sync_ids', table => { + userColumn(table) + table.string('sourceStorageIdentityKey', 130).notNullable() + table.string('entity', 32).notNullable() + table.bigInteger('incomingId').unsigned().notNullable() + table.bigInteger('localId').unsigned().notNullable() + table.primary(['userId', 'sourceStorageIdentityKey', 'entity', 'incomingId']) + }) + } + if (!(await knex.schema.hasTable('snapshot_sync_primary_epochs'))) { + await knex.schema.createTable('snapshot_sync_primary_epochs', table => { + userColumn(table).primary() + table.bigInteger('epoch').notNullable() + }) + } + // A primary may change away and back to the same identity. A database-owned + // counter fences that ABA transition, including older/independent writers. + if (mysql) { + await runInSeries(['snapshot_sync_sessions', 'snapshot_sync_ids', 'snapshot_sync_primary_epochs'], async table => { + const constraint = table + '_user' + const [existing]: Array> = await knex.raw( + "SELECT CONSTRAINT_NAME AS name FROM information_schema.TABLE_CONSTRAINTS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND CONSTRAINT_NAME = ? AND CONSTRAINT_TYPE = 'FOREIGN KEY'", + [table, constraint] + ) + if (existing.length === 0) + await knex.schema.table(table, definition => { + definition.foreign('userId', constraint).references('userId').inTable('users') + }) + }) + const [triggers]: Array> = await knex.raw( + 'SELECT TRIGGER_NAME AS name FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA = DATABASE() AND TRIGGER_NAME = ?', + ['snapshot_sync_primary_change'] + ) + if (triggers.length === 0) + await knex.raw(`CREATE TRIGGER snapshot_sync_primary_change AFTER UPDATE ON users + FOR EACH ROW BEGIN IF NOT (OLD.activeStorage <=> NEW.activeStorage) THEN + INSERT INTO snapshot_sync_primary_epochs (userId, epoch) VALUES (NEW.userId, 1) + ON DUPLICATE KEY UPDATE epoch = epoch + 1; + END IF; END`) + } else { + await knex.raw(`CREATE TRIGGER IF NOT EXISTS snapshot_sync_primary_change AFTER UPDATE OF activeStorage ON users + WHEN OLD.activeStorage IS NOT NEW.activeStorage BEGIN + INSERT INTO snapshot_sync_primary_epochs (userId, epoch) VALUES (NEW.userId, 1) + ON CONFLICT(userId) DO UPDATE SET epoch = epoch + 1; + END`) + } +} + +export async function removeSnapshotSyncTables(knex: Knex): Promise { + await knex.raw('DROP TRIGGER IF EXISTS snapshot_sync_primary_change') + await knex.schema.dropTableIfExists('snapshot_sync_primary_epochs') + await knex.schema.dropTableIfExists('snapshot_sync_ids') + await knex.schema.dropTableIfExists('snapshot_sync_sessions') +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/ConcurrentSnapshotArchiveSource.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/ConcurrentSnapshotArchiveSource.test.ts new file mode 100644 index 000000000..cad78b74d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/ConcurrentSnapshotArchiveSource.test.ts @@ -0,0 +1,373 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { seedArchiveClosure } from '../../../test/utils/snapshotArchiveFixtures' +import { KnexSnapshotArchiveStore } from './archive/KnexSnapshotArchiveStore' +import { KnexSnapshotArchiveRequestStore } from './archive/KnexSnapshotArchiveRequestStore' +import { snapshotArchiveRequestId } from './archive/SnapshotArchiveRequest' +import { captureSnapshotArchiveSource } from './archive/captureSnapshotArchiveSource' +import { verifySnapshotArchiveDirectory } from './archive/SnapshotArchiveDirectory' +import * as ArchiveGuard from './archive/SnapshotArchiveGuard' +import { SnapshotArchiveSourceCleanupError } from './archive/KnexSnapshotArchiveSource' + +const identity = '02' + '11'.repeat(32) +const stores: StorageKnex[] = [] +const directories: string[] = [] +function gate() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'owned-archive-source-')) + directories.push(directory) + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) + stores.push(storage) + await storage.knex.raw('PRAGMA journal_mode = WAL') + await storage.migrate('original source', 'original-source') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identity) + const { user: other } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(storage, user.userId, other.userId) + return storage +} +afterEach(async () => { + jest.restoreAllMocks() + await Promise.all(stores.splice(0).map(store => store.destroy())) + await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))) +}) + +test('a ready request waits for owned reader destruction while foreground storage remains usable', async () => { + const storage = await fixture() + expect(await storage.supportsSnapshotArchiveSource()).toBe(true) + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + const requests = new KnexSnapshotArchiveRequestStore(storage.knex) + const archives = new KnexSnapshotArchiveStore(storage.knex) + const fields = { + version: 1 as const, + nonce: 'b'.repeat(64), + notAfter: Date.now() + 300000, + maxBytes: 32768 + } + const input = { ...fields, requestId: snapshotArchiveRequestId(fields) } + const admitted = await requests.claim(identity, input) + const source = (await storage.openSnapshotArchiveSource(identity))! + expect(source.user.identityKey).toBe(identity) + expect(source.sourceStorage.storageIdentityKey).toBe('original-source') + expect(source.sourceSchema).toBe('2026-10-02-001 repair snapshot SQLite conflict maintenance') + const reader = Reflect.get(storage, 'snapshotSyncSource') as StorageKnex + expect(reader.knex).not.toBe(storage.knex) + expect(reader.knex.client.config.pool).toMatchObject({ min: 0, max: 1 }) + const entered = gate() + const finish = gate() + const destroy = reader.destroy.bind(reader) + const intercept = jest.spyOn(reader, 'destroy').mockImplementation(async () => { + entered.resolve() + await finish.promise + await destroy() + }) + let settled = false + const pending = captureSnapshotArchiveSource( + source, + { + begin: binding => requests.begin(admitted.owner!, binding), + append: (writer, page) => archives.append(writer, page), + seal: writer => requests.seal(admitted.owner!, writer), + close: () => requests.close(identity, input.requestId, 'failed') + }, + identity, + 'test' + ).finally(() => { + settled = true + }) + void pending.catch(() => undefined) + try { + await Promise.race([ + entered.promise, + pending.then(() => { + throw new Error('Capture completed before physical close') + }) + ]) + expect(settled).toBe(false) + expect((await requests.status(identity, input.requestId)).state).toBe('building') + expect((await storage.knex('snapshot_archives').first()).state).toBe('building') + await expect(storage.getSnapshotSync()!.openSource(identity)).rejects.toThrow('opening or active') + await storage.knex('tx_labels').where({ txLabelId: 1 }).update({ label: 'foreground write after capture' }) + finish.resolve() + const manifest = await pending + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + expect((await requests.status(identity, input.requestId)).state).toBe('ready') + const verified = verifySnapshotArchiveDirectory(await archives.directory(identity, manifest.archiveId), { + identityKey: identity, + chain: 'test', + sourceStorageIdentityKey: 'original-source', + digest: manifest.digest + }) + expect(verified.manifest).toEqual(manifest) + const next = (await storage.getSnapshotSync()!.openSource(identity))! + await next.close() + await requests.close(identity, input.requestId) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) + } finally { + finish.resolve() + await pending.catch(() => undefined) + intercept.mockRestore() + } +}) + +test('provider destruction closes its archive reader and fences further admission', async () => { + const storage = await fixture() + const source = (await storage.openSnapshotArchiveSource(identity))! + await storage.destroy() + await source.closed + expect(source.isOpen).toBe(false) + expect(await storage.supportsSnapshotArchiveSource()).toBe(false) + await expect(storage.openSnapshotArchiveSource(identity)).rejects.toThrow('destruction begins') +}) + +test('cancellation after source acquisition closes the owned pool before any archive is reserved', async () => { + const storage = await fixture() + const source = (await storage.openSnapshotArchiveSource(identity))! + const controller = new AbortController() + controller.abort() + await expect( + captureSnapshotArchiveSource(source, new KnexSnapshotArchiveStore(storage.knex), identity, 'test', { + signal: controller.signal + }) + ).rejects.toThrow('cancelled') + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + expect(await storage.knex('snapshot_archives')).toHaveLength(0) +}) + +test('unsupported in-memory SQLite does not construct a dedicated archive reader', async () => { + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + }) + stores.push(storage) + expect(await storage.supportsSnapshotArchiveSource()).toBe(false) + expect(await storage.openSnapshotArchiveSource(identity)).toBeUndefined() + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() +}) + +test('one admitted recovery flight remains drained by destruction while foreground reads continue', async () => { + const storage = await fixture() + const entered = gate() + const finish = gate() + const recover = ArchiveGuard.recoverSnapshotArchiveGuards + const recovery = jest.spyOn(ArchiveGuard, 'recoverSnapshotArchiveGuards').mockImplementation(async (...args) => { + entered.resolve() + await finish.promise + await recover(...args) + }) + const destroy = jest.spyOn(storage.knex.client, 'destroy') + const first = storage.recoverSnapshotArchiveSources() + const second = storage.recoverSnapshotArchiveSources() + expect(second).toBe(first) + let destruction: Promise | undefined + try { + await entered.promise + expect((await storage.findUsers({ partial: { identityKey: identity } }))[0].identityKey).toBe(identity) + destruction = storage.destroy() + await expect(storage.recoverSnapshotArchiveSources()).rejects.toThrow('after destruction') + expect(destroy).not.toHaveBeenCalled() + expect(recovery).toHaveBeenCalledTimes(1) + finish.resolve() + await Promise.all([first, destruction]) + expect(destroy).toHaveBeenCalledTimes(1) + await storage.awaitSnapshotArchiveRecovery() + } finally { + finish.resolve() + await Promise.allSettled([first, destruction]) + } +}) + +test('failed recovery preserves its error and releases the flight for a later retry', async () => { + const storage = await fixture() + const failure = new Error('synthetic recovery enumeration failure') + const recovery = jest.spyOn(ArchiveGuard, 'recoverSnapshotArchiveGuards').mockRejectedValueOnce(failure) + await expect(storage.recoverSnapshotArchiveSources()).rejects.toBe(failure) + await storage.recoverSnapshotArchiveSources() + expect(recovery).toHaveBeenCalledTimes(2) + await storage.awaitSnapshotArchiveRecovery() + expect(await storage.supportsSnapshotArchiveSource()).toBe(true) +}) + +test('destruction drains a failing admitted recovery and still destroys the foreground pool', async () => { + const storage = await fixture() + const entered = gate() + const finish = gate() + const failure = new Error('synthetic admitted recovery failure') + jest.spyOn(ArchiveGuard, 'recoverSnapshotArchiveGuards').mockImplementation(async () => { + entered.resolve() + await finish.promise + throw failure + }) + const destroy = jest.spyOn(storage.knex.client, 'destroy') + const recovery = storage.recoverSnapshotArchiveSources() + void recovery.catch(() => undefined) + let destruction: Promise | undefined + try { + await entered.promise + destruction = storage.destroy() + void destruction.catch(() => undefined) + expect(destroy).not.toHaveBeenCalled() + finish.resolve() + await expect(recovery).rejects.toBe(failure) + await expect(destruction).rejects.toBe(failure) + expect(destroy).toHaveBeenCalledTimes(1) + await expect(storage.recoverSnapshotArchiveSources()).rejects.toThrow('after destruction') + } finally { + finish.resolve() + await Promise.allSettled([recovery, destruction]) + } +}) + +test('unsupported in-memory recovery performs no guard work and idle drainage is safe', async () => { + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + }) + stores.push(storage) + const recovery = jest.spyOn(ArchiveGuard, 'recoverSnapshotArchiveGuards') + await storage.awaitSnapshotArchiveRecovery() + await storage.recoverSnapshotArchiveSources() + expect(recovery).not.toHaveBeenCalled() +}) + +test('the static MySQL source probe does not construct or acquire a private connection', async () => { + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'mysql2', + connection: { + host: '127.0.0.1', + port: 1, + database: 'synthetic_unconnected_probe' + }, + pool: { min: 0, max: 1 } + }) + }) + stores.push(storage) + const acquire = jest.spyOn(storage.knex.client, 'acquireConnection').mockRejectedValue(new Error('Unexpected I/O')) + expect(await storage.supportsSnapshotArchiveSource()).toBe(true) + expect(acquire).not.toHaveBeenCalled() + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() +}) + +test.each(['opening', 'closing'] as const)( + 'unproved guarded source cleanup during %s preserves ownership and permanently fences its provider', + async phase => { + const storage = await fixture() + const requests = new KnexSnapshotArchiveRequestStore(storage.knex, true, true) + const offered = await requests.offer(identity, { + lifetimeMs: 300000, + maxBytes: 32768 + }) + const { owner } = await requests.claimReader(identity, offered.request) + const failure = new SnapshotArchiveSourceCleanupError(new Error('synthetic native cleanup outcome')) + const read = ArchiveGuard.readGuardedSnapshotArchive + let native: { open: boolean; close: () => void } | undefined + if (phase === 'opening') jest.spyOn(ArchiveGuard, 'readGuardedSnapshotArchive').mockRejectedValueOnce(failure) + else { + jest.spyOn(ArchiveGuard, 'readGuardedSnapshotArchive').mockImplementationOnce(async (...args) => { + jest.spyOn(args[1].client, 'destroyRawConnection').mockImplementation(async connection => { + native = connection + }) + return await read(...args) + }) + } + let received: unknown + try { + try { + const source = (await storage.openSnapshotArchiveSource(identity, {}, owner))! + await source.close() + } catch (error) { + received = error + } + expect(received).toBeInstanceOf(SnapshotArchiveSourceCleanupError) + if (phase === 'opening') expect(received).toBe(failure) + else expect(native?.open).toBe(true) + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(1) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeDefined() + expect(await storage.supportsSnapshotArchiveSource()).toBe(false) + await expect(storage.openSnapshotArchiveSource(identity, {}, owner)).rejects.toThrow() + await expect(storage.getSnapshotSync()!.openSource(identity)).rejects.toThrow('destruction begins') + await expect(storage.recoverSnapshotArchiveSources()).rejects.toThrow('after destruction') + await expect(storage.destroy()).rejects.toBe(received) + } finally { + jest.restoreAllMocks() + // Only the synthetic fixture may prove and close this intentionally retained native connection. + if (native?.open) native.close() + stores.splice(stores.indexOf(storage), 1) + await storage.destroy().catch(error => { + expect(error).toBe(received) + }) + } + } +) + +test.each([new Error('synthetic guarded read failure'), undefined])( + 'an ordinary guarded read failure %p preserves its error but releases a physically cleaned reader', + async failure => { + const storage = await fixture() + const requests = new KnexSnapshotArchiveRequestStore(storage.knex, true, true) + const offered = await requests.offer(identity, { + lifetimeMs: 300000, + maxBytes: 32768 + }) + const { owner } = await requests.claimReader(identity, offered.request) + jest.spyOn(ArchiveGuard, 'readGuardedSnapshotArchive').mockRejectedValueOnce(failure) + await expect(storage.openSnapshotArchiveSource(identity, {}, owner)).rejects.toBe(failure) + expect(await storage.supportsSnapshotArchiveSource()).toBe(true) + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + const source = (await storage.openSnapshotArchiveSource(identity, {}, owner))! + expect(source.user.identityKey).toBe(identity) + await source.close() + await requests.sourceClosed(owner!) + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(0) + } +) + +test('destroying an active guarded reader preserves its ordinary read error for the consumer after proved cleanup', async () => { + const storage = await fixture() + const requests = new KnexSnapshotArchiveRequestStore(storage.knex, true, true) + const offered = await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }) + const { owner } = await requests.claimReader(identity, offered.request) + const failure = new Error('synthetic read completion failure after native closure') + const read = ArchiveGuard.readGuardedSnapshotArchive + jest.spyOn(ArchiveGuard, 'readGuardedSnapshotArchive').mockImplementationOnce(async (...args) => { + await read(...args) + throw failure + }) + const source = (await storage.openSnapshotArchiveSource(identity, {}, owner))! + const reader = Reflect.get(storage, 'snapshotSyncSource') as StorageKnex + const consumer = source.closed.catch(error => error) + await expect(reader.destroy()).resolves.toBeUndefined() + expect(await consumer).toBe(failure) + await expect(reader.knex.raw('SELECT 1')).rejects.toThrow('Unable to acquire a connection') + await requests.sourceClosed(owner!) + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(0) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/ConcurrentSnapshotSyncSource.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/ConcurrentSnapshotSyncSource.test.ts new file mode 100644 index 000000000..2a24a1cce --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/ConcurrentSnapshotSyncSource.test.ts @@ -0,0 +1,284 @@ +import { knex } from 'knex' +import { StorageKnex, type StorageKnexOptions } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import type { WalletReadSnapshot } from './WalletReadSnapshot' + +function gate() { + let resolve!: () => void + const promise = new Promise(yes => { + resolve = yes + }) + return { promise, resolve } +} +const pendingOperations: Promise[] = [] +function observe(operation: Promise): Promise { + // Keep secondary cleanup failures attached when a preceding fault assertion + // fails. Every primary result is still awaited/asserted by its owning test. + void operation.catch(() => undefined) + pendingOperations.push(operation) + return operation +} +async function waitForBoundary(boundary: Promise, operation: Promise): Promise { + await Promise.race([ + boundary, + operation.then(() => { + throw new Error('Operation completed before the required concurrency boundary') + }) + ]) +} +const stores: StorageKnex[] = [] +function mysql(telemetry?: StorageKnexOptions['telemetry']) { + const store = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + telemetry, + knex: knex({ + client: 'mysql2', + connection: { host: '127.0.0.1', database: 'synthetic', user: 'unit', password: 'synthetic-only' }, + pool: { min: 0, max: 1 } + }) + }) + stores.push(store) + return store +} +function view(): WalletReadSnapshot { + const closed = gate() + let open = true + return { + version: 1, + snapshotId: 'a'.repeat(64), + sourceStorage: {} as WalletReadSnapshot['sourceStorage'], + user: {} as WalletReadSnapshot['user'], + expiresAt: Date.now() + 10000, + get isOpen() { + return open + }, + closed: closed.promise, + close: async () => { + open = false + closed.resolve() + }, + readPage: jest.fn() + } +} +afterEach(async () => { + jest.restoreAllMocks() + for (const store of stores.splice(0)) await store.destroy() + await Promise.allSettled(pendingOperations.splice(0)) +}) + +test('the dedicated MySQL reader preserves hidden credentials and leaves original pool capacity intact', async () => { + const capture = jest.fn() + let enabled = true + const source = mysql({ sink: { capture }, enabled: () => enabled }) + const result = view() + const opening = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockImplementation(async function ( + this: StorageKnex + ) { + const connection = this.knex.client.config.connection as object + const password = Object.getOwnPropertyDescriptor(connection, 'password')! + expect(password.value).toBe('synthetic-only') + expect(password.enumerable).toBe(false) + expect(this.knex.client.config.pool.max).toBe(1) + expect(this.getSnapshotSync()).toBeUndefined() + this.knex.emit('query', { __knexQueryUid: 'synthetic', method: 'select', sql: 'private data' }) + this.knex.emit('query-response', [], { __knexQueryUid: 'synthetic' }) + return result + }) + const opened = (await source.getSnapshotSync()!.openSource('identity'))! + const child = opening.mock.contexts[0] as StorageKnex + expect(child).not.toBe(source) + expect(source.knex.client.config.pool.max).toBe(1) + expect(capture).toHaveBeenCalled() + expect(JSON.stringify(capture.mock.calls)).not.toContain('private data') + enabled = false + expect(child.telemetry.enabled).toBe(false) + const cleanup = jest.spyOn(child, 'destroy') + await opened.close() + expect(cleanup).toHaveBeenCalledTimes(1) + expect(opened.isOpen).toBe(false) +}) + +test('dynamic connection providers and externally shared pools retain the legacy path', async () => { + const source = mysql() + const original = source.knex.client.config.connection + const open = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot') + source.knex.client.config.connection = async () => original + expect(await source.getSnapshotSync()!.openSource('identity')).toBeUndefined() + source.knex.client.config.connection = original + source.knex.client.config.connectionPool = { externallyOwned: true } + expect(await source.getSnapshotSync()!.openSource('identity')).toBeUndefined() + delete source.knex.client.config.connectionPool + source.knex.client.config.connection = { user: 'unit' } + expect(await source.getSnapshotSync()!.openSource('identity')).toBeUndefined() + expect(open).not.toHaveBeenCalled() +}) + +test('provider shutdown completes unfinished database telemetry once with cancelled status', async () => { + const capture = jest.fn() + const source = mysql({ sink: { capture }, enabled: true }) + source.knex.emit('query', { + __knexQueryUid: 'synthetic-unfinished-query', + method: 'select', + sql: 'private fixture payload' + }) + expect(capture).not.toHaveBeenCalled() + await source.destroy() + stores.splice(stores.indexOf(source), 1) + expect(capture).toHaveBeenCalledTimes(1) + expect(capture.mock.calls[0][0]).toMatchObject({ name: 'wallet.storage.db.query', spanStatus: 'cancelled' }) + expect(JSON.stringify(capture.mock.calls)).not.toContain('private fixture payload') + source.knex.emit('query-response', [], { __knexQueryUid: 'synthetic-unfinished-query' }) + await source.destroy() + expect(capture).toHaveBeenCalledTimes(1) +}) + +test.each([undefined, '', ':memory:', 'file:synthetic-snapshot'])( + 'SQLite filename %s cannot open an independent retained reader', + async filename => { + const source = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ client: 'better-sqlite3', connection: { filename: filename as string }, useNullAsDefault: true }) + }) + stores.push(source) + const opening = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot') + const query = jest.spyOn(source.knex.client, 'raw') + expect(await source.getSnapshotSync()!.openSource('identity')).toBeUndefined() + expect(opening).not.toHaveBeenCalled() + expect(query).not.toHaveBeenCalled() + } +) + +test('destroy fences a source that has not finished opening and drains its private reader', async () => { + const source = mysql() + const waiting = gate() + const started = gate() + const result = view() + const closing = jest.spyOn(result, 'close') + jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockImplementation(async () => { + started.resolve() + await waiting.promise + return result + }) + const opening = observe(source.getSnapshotSync()!.openSource('identity')) + let destroying: Promise | undefined + try { + await waitForBoundary(started.promise, opening) + destroying = observe(source.destroy()) + } finally { + waiting.resolve() + } + await expect(opening).rejects.toThrow('destruction') + await destroying + expect(closing).toHaveBeenCalled() + expect(result.isOpen).toBe(false) +}) + +test('physical reader cleanup retains the admission slot until it finishes', async () => { + const source = mysql() + const result = view() + const waiting = gate() + const cleaning = gate() + const opening = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockResolvedValue(result) + const opened = (await source.getSnapshotSync()!.openSource('identity'))! + const child = opening.mock.contexts[0] as StorageKnex + const original = child.destroy.bind(child) + jest.spyOn(child, 'destroy').mockImplementation(async () => { + cleaning.resolve() + await waiting.promise + await original() + }) + const closing = observe(opened.close()) + try { + await waitForBoundary(cleaning.promise, closing) + await expect(source.getSnapshotSync()!.openSource('identity')).rejects.toThrow('already has') + } finally { + waiting.resolve() + } + await closing +}) + +test('failed physical cleanup fences further source admission', async () => { + const source = mysql() + const result = view() + const opening = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockResolvedValue(result) + const opened = (await source.getSnapshotSync()!.openSource('identity'))! + const child = opening.mock.contexts[0] as StorageKnex + const cleanup = jest.spyOn(child, 'destroy').mockRejectedValue(new Error('synthetic cleanup failure')) + await expect(opened.close()).rejects.toThrow('synthetic cleanup failure') + await expect(source.getSnapshotSync()!.openSource('identity')).rejects.toThrow('destruction') + cleanup.mockRestore() + await child.destroy() +}) + +test('monotonic setup expiry releases admission before allocating a reader even if wall time is unchanged', async () => { + const source = mysql() + const open = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockResolvedValue(view()) + const wall = jest.spyOn(Date, 'now').mockReturnValue(2000000) + const monotonic = jest.spyOn(performance, 'now').mockReturnValueOnce(1).mockReturnValue(1001) + await expect(source.getSnapshotSync()!.openSource('identity', { lifetimeMs: 1000 })).rejects.toThrow( + 'expired during connection setup' + ) + expect(open).not.toHaveBeenCalled() + monotonic.mockRestore() + wall.mockRestore() + const fresh = (await source.getSnapshotSync()!.openSource('identity'))! + await fresh.close() + expect(open).toHaveBeenCalledTimes(1) +}) + +test.each([1, 3600000])( + 'the exact lifetime boundary %s and cancellation signal reach the dedicated reader', + async lifetimeMs => { + const source = mysql() + const controller = new AbortController() + jest.spyOn(Date, 'now').mockReturnValue(5000) + jest.spyOn(performance, 'now').mockReturnValue(100) + const open = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockResolvedValue(view()) + const opened = (await source.getSnapshotSync()!.openSource('identity', { lifetimeMs, signal: controller.signal }))! + expect(open).toHaveBeenCalledWith('identity', { lifetimeMs, signal: controller.signal }) + await opened.close() + } +) + +test('wall-clock setup expiry also fences opening when the monotonic clock does not advance', async () => { + const source = mysql() + jest.spyOn(performance, 'now').mockReturnValue(100) + jest.spyOn(Date, 'now').mockReturnValueOnce(5000).mockReturnValue(6000) + const open = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot') + await expect(source.getSnapshotSync()!.openSource('identity', { lifetimeMs: 1000 })).rejects.toThrow( + 'expired during connection setup' + ) + expect(open).not.toHaveBeenCalled() +}) + +test.each([null, 7])('unsupported connection value %p refuses before constructing another reader', async connection => { + const source = mysql() + const original = source.knex.client.config.connection + source.knex.client.config.connection = connection + const open = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot') + try { + expect(await source.getSnapshotSync()!.openSource('identity')).toBeUndefined() + expect(open).not.toHaveBeenCalled() + } finally { + source.knex.client.config.connection = original + } +}) + +test('invalid static MySQL database metadata never creates a private reader', async () => { + const source = mysql() + const opening = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockResolvedValue(view()) + for (const database of [null, 42, undefined]) { + source.knex.client.config.connection = { host: '127.0.0.1', database } + expect(await source.getSnapshotSync()!.openSource('identity')).toBeUndefined() + } + expect(opening).not.toHaveBeenCalled() +}) + +test('a pre-cancelled source does no database or reader work', async () => { + const source = mysql() + const opening = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot').mockResolvedValue(view()) + const signal = new AbortController() + signal.abort() + await expect(source.getSnapshotSync()!.openSource('identity', { signal: signal.signal })).rejects.toThrow('cancelled') + expect(opening).not.toHaveBeenCalled() +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexSnapshotSyncDestination.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexSnapshotSyncDestination.ts new file mode 100644 index 000000000..025ecda3a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexSnapshotSyncDestination.ts @@ -0,0 +1,318 @@ +import { SNAPSHOT_SYNC_MIGRATION } from '../schema/snapshotSyncMigration' +import { Random, Utils } from '@bsv/sdk' +import type { StorageKnex } from '../StorageKnex' +import type { SyncChunk, TrxToken } from '../../sdk/WalletStorage.interfaces' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +import { verifyOne } from '../../utility/utilityHelpers' +import { runInSeries } from '../../utility/runInSeries' +import type { TableProvenTx, TableUser } from '../schema/tables' +import { mergeSyncChunkEntities } from '../schema/entities/mergeSyncChunkEntities' +import { sameSyncProof } from '../methods/validateSyncProof' +import { detachSnapshotSyncPage, loadSnapshotIdMap } from './SnapshotSyncRows' +import { + snapshotSyncTables, + type SnapshotSyncCheckpoint, + type SnapshotSyncCommit, + type SnapshotSyncSource, + type SnapshotSyncTable +} from './SnapshotSync' +import type { WalletSnapshotPage } from './WalletReadSnapshot' +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import { copySnapshotCursor, sameSnapshotArchivePosition } from './SnapshotCursor' + +interface SessionRow { + version: 1 + userId: number + identityKey: string + sourceStorageIdentityKey: string + destinationStorageIdentityKey: string + sessionId: string + snapshotId: string + sourceUserId: number + chain: string + primaryEpoch: number + sourceActiveStorage: string | null + sourceUserCreatedAt: string + sourceUserUpdatedAt: string + activeStorage: string | null + expiresAt: number + sequence: number + tableIndex: number + cursor: string | null +} + +type ProofPreparation = (chunk: SyncChunk) => Promise> + +function checkpoint(row: SessionRow): SnapshotSyncCheckpoint { + return { + version: row.version, + sessionId: row.sessionId, + identityKey: row.identityKey, + sourceStorageIdentityKey: row.sourceStorageIdentityKey, + destinationStorageIdentityKey: row.destinationStorageIdentityKey, + snapshotId: row.snapshotId, + tableIndex: row.tableIndex, + sequence: row.sequence, + cursor: row.cursor == null ? undefined : JSON.parse(row.cursor), + done: row.tableIndex === snapshotSyncTables.length + } +} + +function notMatchingCheckpoint(actual: SnapshotSyncCheckpoint, expected: SnapshotSyncCheckpoint): boolean { + return ( + actual.version !== expected.version || + actual.sessionId !== expected.sessionId || + actual.identityKey !== expected.identityKey || + actual.sourceStorageIdentityKey !== expected.sourceStorageIdentityKey || + actual.destinationStorageIdentityKey !== expected.destinationStorageIdentityKey || + actual.snapshotId !== expected.snapshotId || + actual.tableIndex !== expected.tableIndex || + actual.sequence !== expected.sequence || + actual.done !== expected.done || + actual.cursor?.version !== expected.cursor?.version || + actual.cursor?.snapshotId !== expected.cursor?.snapshotId || + actual.cursor?.table !== expected.cursor?.table || + JSON.stringify(actual.cursor?.after) !== JSON.stringify(expected.cursor?.after) || + !sameSnapshotArchivePosition(actual.cursor?.archivePosition, expected.cursor?.archivePosition) + ) +} + +function requireLiveSource(expiresAt: number, stage: 'session admission' | 'destination commit'): void { + if (Date.now() >= expiresAt) throw new SnapshotResourceLimitError(`Snapshot source expired before ${stage}`) +} + +function validateSource(source: SnapshotSyncSource): void { + if ( + source.version !== 1 || + !/^[a-f0-9]{64}$/.test(source.snapshotId) || + !/^(02|03)[a-f0-9]{64}$/i.test(source.user.identityKey) || + !Number.isSafeInteger(source.user.userId) || + source.user.userId < 1 || + !Number.isSafeInteger(source.expiresAt) || + source.expiresAt > Date.now() + 3600000 || + typeof source.sourceStorage.storageIdentityKey !== 'string' || + source.sourceStorage.storageIdentityKey.length === 0 || + source.sourceStorage.storageIdentityKey.length > 130 + ) { + throw new WERR_INVALID_PARAMETER('source', 'a live version-one wallet snapshot with a bound profile and source') + } + requireLiveSource(source.expiresAt, 'session admission') +} + +/** Destination-authoritative state. Every session mutation locks the user before its session to preserve one lock order. */ +export class KnexSnapshotSyncDestination { + constructor( + private readonly storage: StorageKnex, + private readonly prepareProofs: ProofPreparation + ) {} + + async supportsDestination(): Promise { + const k = this.storage.knex + if (!(await k.schema.hasTable('knex_migrations'))) return false + return (await k('knex_migrations').select('name').where({ name: SNAPSHOT_SYNC_MIGRATION }).first()) !== undefined + } + + private async lockUser(identityKey: string, trx: TrxToken): Promise { + const k = this.storage.toDb(trx) + // Acquire the write lock before reading on both SQLite (deferred transaction) + // and MySQL. Primary-selection updates and sibling processes use this same row. + await k('users') + .where({ identityKey }) + .update({ userId: k.ref('userId') }) + return verifyOne(await this.storage.findUsers({ partial: { identityKey }, trx })) + } + + private async primaryEpoch(userId: number, trx: TrxToken): Promise { + const row: { epoch: number } | undefined = await this.storage + .toDb(trx)('snapshot_sync_primary_epochs') + .where({ userId }) + .first() + const epoch = Number(row?.epoch ?? 0) + if (!Number.isSafeInteger(epoch) || epoch < 0) throw new WERR_INVALID_OPERATION('Invalid primary epoch') + return epoch + } + + async checkpoint(identityKey: string, sourceStorageIdentityKey: string): Promise { + const user = await this.storage.findUserByIdentityKey(identityKey) + if (user === undefined) return undefined + const row: SessionRow | undefined = await this.storage + .knex('snapshot_sync_sessions') + .where({ userId: user.userId, sourceStorageIdentityKey }) + .first() + return row === undefined ? undefined : checkpoint(row) + } + + async begin(input: SnapshotSyncSource, activeStorage: string | undefined): Promise { + // Snapshot all caller-controlled values before the first asynchronous boundary. + validateSource(input) + const source = { + ...input, + sourceStorage: { ...input.sourceStorage }, + user: { + ...input.user, + created_at: new Date(input.user.created_at), + updated_at: new Date(input.user.updated_at), + activeStorage + } + } + const sourceUserCreatedAt = source.user.created_at.toISOString() + const sourceUserUpdatedAt = source.user.updated_at.toISOString() + if (!(await this.supportsDestination())) + throw new WERR_INVALID_OPERATION('Snapshot sync destination requires the version-one migration') + const settings = await this.storage.makeAvailable() + if ( + source.sourceStorage.chain !== settings.chain || + source.sourceStorage.storageIdentityKey === settings.storageIdentityKey + ) { + throw new WERR_INVALID_PARAMETER('source', 'a different storage on the destination network') + } + await this.storage.findOrInsertUser(source.user.identityKey) + return await this.storage.transaction(async trx => { + const user = await this.lockUser(source.user.identityKey, trx) + const k = this.storage.toDb(trx) + const scope = { userId: user.userId, sourceStorageIdentityKey: source.sourceStorage.storageIdentityKey } + const previous: SessionRow | undefined = await k('snapshot_sync_sessions').where(scope).first() + if (previous?.snapshotId === source.snapshotId) { + if ( + previous.sourceUserId !== source.user.userId || + previous.sourceActiveStorage !== (activeStorage ?? null) || + previous.sourceUserCreatedAt !== sourceUserCreatedAt || + previous.sourceUserUpdatedAt !== sourceUserUpdatedAt || + previous.chain !== settings.chain || + previous.destinationStorageIdentityKey !== settings.storageIdentityKey || + previous.version !== source.version || + previous.activeStorage !== (user.activeStorage ?? null) || + Number(previous.primaryEpoch) !== (await this.primaryEpoch(user.userId, trx)) || + Number(previous.expiresAt) !== source.expiresAt + ) { + throw new WERR_INVALID_OPERATION('Snapshot session binding changed or expired; open a new source view') + } + requireLiveSource(Number(previous.expiresAt), 'session admission') + return checkpoint(previous) + } + requireLiveSource(source.expiresAt, 'session admission') + // Profile metadata is merged with the first page, so cancellation before + // any page leaves the selected primary untouched and update counts agree + // with the durable merge outcome. + const row: SessionRow = { + ...scope, + version: 1, + sessionId: Utils.toHex(Random(32)), + snapshotId: source.snapshotId, + sourceUserId: source.user.userId, + identityKey: source.user.identityKey, + destinationStorageIdentityKey: settings.storageIdentityKey, + chain: settings.chain, + activeStorage: user.activeStorage ?? null, + sourceActiveStorage: activeStorage ?? null, + sourceUserCreatedAt, + sourceUserUpdatedAt, + primaryEpoch: await this.primaryEpoch(user.userId, trx), + expiresAt: source.expiresAt, + tableIndex: 0, + sequence: 0, + cursor: null + } + await k('snapshot_sync_sessions').insert(row).onConflict(['userId', 'sourceStorageIdentityKey']).merge(row) + return checkpoint(row) + }) + } + + async prepare( + input: SnapshotSyncCheckpoint, + page: WalletSnapshotPage + ): Promise<() => Promise> { + const expected: SnapshotSyncCheckpoint = { + ...input, + cursor: copySnapshotCursor(input.cursor) + } + if ( + expected.version !== 1 || + !/^[a-f0-9]{64}$/.test(expected.sessionId) || + !Number.isSafeInteger(expected.sequence) || + expected.sequence < 0 || + !Number.isSafeInteger(expected.tableIndex) || + expected.tableIndex < 0 + ) { + throw new WERR_INVALID_PARAMETER('checkpoint', 'a version-one durable snapshot checkpoint') + } + const detached = detachSnapshotSyncPage(expected, page) + const proofs = await this.prepareProofs(detached.chunk) + let consumed = false + return async () => { + if (consumed) + throw new WERR_INVALID_OPERATION('Prepared snapshot page already consumed; read its durable checkpoint') + consumed = true + return await this.storage.transaction(async trx => { + const user = await this.lockUser(expected.identityKey, trx) + const k = this.storage.toDb(trx) + const scope = { userId: user.userId, sourceStorageIdentityKey: expected.sourceStorageIdentityKey } + const row: SessionRow | undefined = await k('snapshot_sync_sessions').where(scope).first() + if ( + row === undefined || + notMatchingCheckpoint(checkpoint(row), expected) || + row.activeStorage !== (user.activeStorage ?? null) || + Number(row.primaryEpoch) !== (await this.primaryEpoch(user.userId, trx)) || + row.chain !== this.storage.chain || + row.destinationStorageIdentityKey !== this.storage.getSettings().storageIdentityKey + ) { + throw new WERR_INVALID_OPERATION('Snapshot session changed or expired; resume from its durable checkpoint') + } + requireLiveSource(Number(row.expiresAt), 'destination commit') + await this.verifyProofs(proofs, trx) + const mappings = await loadSnapshotIdMap( + k, + scope, + row.sourceUserId, + snapshotSyncTables[row.tableIndex], + detached.chunk + ) + if (row.sequence === 0) + detached.chunk.user = { + userId: row.sourceUserId, + identityKey: row.identityKey, + created_at: new Date(row.sourceUserCreatedAt), + updated_at: new Date(row.sourceUserUpdatedAt), + // TableUser retains its legacy declaration although an unselected + // primary is represented as undefined by storage adapters. + activeStorage: (row.sourceActiveStorage ?? undefined) as string + } + const result = await mergeSyncChunkEntities( + this.storage, + user.userId, + undefined, + detached.chunk, + mappings.map, + trx + ) + await mappings.persist() + const currentUser = verifyOne(await this.storage.findUsers({ partial: { identityKey: row.identityKey }, trx })) + row.primaryEpoch = await this.primaryEpoch(user.userId, trx) + row.activeStorage = currentUser.activeStorage ?? null + row.sequence++ + row.tableIndex = detached.nextTable + row.cursor = detached.nextCursor + await k('snapshot_sync_sessions').where(scope).update({ + sequence: row.sequence, + tableIndex: row.tableIndex, + cursor: row.cursor, + primaryEpoch: row.primaryEpoch, + activeStorage: row.activeStorage + }) + return { checkpoint: checkpoint(row), inserts: result.inserts, updates: result.updates } + }) + } + } + + private async verifyProofs(proofs: Map, trx: TrxToken): Promise { + const ordered = [...proofs.values()].sort((left, right) => left.txid.localeCompare(right.txid)) + await runInSeries(ordered, async previous => { + const rows = await this.storage.findProvenTxs({ partial: { txid: previous.txid }, trx }) + if (rows.length !== 1 || !sameSyncProof(rows[0], previous)) { + throw new WERR_INVALID_OPERATION( + 'Proof changed during snapshot preparation; resume from its durable checkpoint' + ) + } + }) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.mysql.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.mysql.test.ts new file mode 100644 index 000000000..51898357a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.mysql.test.ts @@ -0,0 +1,261 @@ +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { walletSnapshotSourceQuery } from './KnexWalletReadSnapshot' + +const identity = '02' + '11'.repeat(32) +const when = new Date('2026-01-01T00:00:00.000Z') + +/** Actual installed Knex MySQL compiler/transaction protocol with a bounded driver response oracle. */ +function fixture(fieldCount = 6, bytes = 524, certificate = false, deleted: boolean | number = 0) { + const db = knex({ client: 'mysql2' }) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: db }) + const queries: Array<{ sql: string; bindings: unknown[] }> = [] + const fields = certificate + ? ['fieldName', 'certificateId', 'userId', 'created_at', 'updated_at', 'fieldValue', 'masterKey'] + : ['txLabelId', 'userId', 'created_at', 'updated_at', 'label', 'isDeleted'] + const table = certificate ? 'certificate_fields' : 'tx_labels' + const fieldName = '😀'.repeat(100) + const connection = { + destroy() {}, + query( + query: { sql: string }, + bindings: unknown[], + callback: (error: Error | null, rows?: unknown[], fields?: unknown[]) => void + ) { + queries.push({ sql: query.sql, bindings }) + if (query.sql.startsWith('select * from information_schema.tables')) { + // This fixture represents an unmigrated externally supplied pool. + expect(query.sql).toBe( + 'select * from information_schema.tables where table_name = ? and table_schema = database()' + ) + expect(bindings).toEqual(['knex_migrations']) + callback(null, [], []) + } else if (query.sql.startsWith('SELECT COLUMN_NAME')) { + expect(query.sql).toBe( + 'SELECT COLUMN_NAME AS name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION LIMIT 65' + ) + expect(bindings).toEqual([table]) + callback( + null, + Array.from({ length: fieldCount }, (_, i) => ({ name: fields[i] ?? `extra${i}` })), + [] + ) + } else if (query.sql.includes('__snapshotBytes')) { + if (certificate) { + expect(query.sql).toContain('case when octet_length(`fieldName`) <= 400 then `fieldName` end as `fieldName`') + expect(query.sql).toContain('`certificate_fields`.`userId` = ?') + const after = query.sql.includes('`fieldName` > ?') + if (after) { + expect(query.sql).toContain('OR (`fieldName` = ? AND `certificateId` > ?)') + expect(bindings.slice(-4)).toEqual([fieldName, fieldName, 1, 128]) + } + callback(null, after ? [] : [{ fieldName, certificateId: 1, __snapshotBytes: bytes, __snapshotOwned: 1 }], []) + } else { + expect(query.sql).toContain('octet_length(`label`)') + expect(query.sql).toContain('`tx_labels`.`userId` = ?') + expect(bindings).toEqual([1, 128]) + callback(null, [{ txLabelId: 1, __snapshotBytes: bytes, __snapshotOwned: 1 }], []) + } + } else if (query.sql.includes('`certificate_fields`.*')) { + callback( + null, + [ + { + fieldName, + certificateId: 1, + userId: 1, + created_at: when, + updated_at: when, + fieldValue: 'value', + masterKey: 'key' + } + ], + [] + ) + } else if (query.sql.includes('`tx_labels`.*')) { + callback( + null, + [{ txLabelId: 1, userId: 1, created_at: when, updated_at: when, label: 'retained', isDeleted: deleted }], + [] + ) + } else if (/^(SET TRANSACTION|BEGIN|COMMIT|ROLLBACK)/.test(query.sql)) callback(null, [], []) + else callback(new Error('Unexpected synthetic driver query')) + } + } + jest.spyOn(db.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(db.client, 'releaseConnection').mockResolvedValue(undefined) + const settings = { + created_at: when, + updated_at: when, + storageIdentityKey: 'synthetic', + storageName: 'synthetic MySQL', + chain: 'test' as const, + dbtype: 'MySQL' as const, + maxOutputScript: 1024 + } + source._settings = settings + jest.spyOn(source, 'makeAvailable').mockResolvedValue(settings) + jest.spyOn(source, 'readSettings').mockResolvedValue(settings) + jest.spyOn(source, 'findUserByIdentityKey').mockResolvedValue({ + created_at: when, + updated_at: when, + userId: 1, + identityKey: identity, + activeStorage: 'synthetic' + }) + return { source, db, queries } +} + +afterEach(() => jest.restoreAllMocks()) + +test('MySQL preflights complete Unicode keys and preserves a boolean-returning pool typecast', async () => { + const { source } = fixture(7, 1352, true) + try { + const view = await source.openWalletReadSnapshot(identity) + const page = await view.readPage('certificateFields') + expect(page.rows[0].fieldName).toBe('😀'.repeat(100)) + expect(page.cursor?.after).toEqual(['😀'.repeat(100), 1]) + expect(await view.readPage('certificateFields', page.cursor)).toMatchObject({ + rows: [], + done: true, + payloadBytes: 0 + }) + await view.close() + } finally { + await source.destroy() + } + const second = fixture(6, 524, false, false) + try { + const view = await second.source.openWalletReadSnapshot(identity) + expect((await view.readPage('txLabels')).rows[0].isDeleted).toBe(false) + await view.close() + } finally { + await second.source.destroy() + } +}) + +test.each([ + ['txLabelMaps', 'tx_labels_map', 'txLabelId', 'transactionId', 0], + ['outputTagMaps', 'output_tags_map', 'outputTagId', 'outputId', 1] +] as const)( + 'MySQL %s uses the cursor-order auxiliary primary key and both map keys', + (table, name, left, right, tableId) => { + const k = knex({ client: 'mysql2' }) + const query = walletSnapshotSourceQuery(k, table, 41, true, true).select(`${name}.*`).toSQL() + expect(query.sql).toContain('from `snapshot_relation_keys` FORCE INDEX (`PRIMARY`) cross join `' + name + '`') + expect(query.sql).toContain( + '`snapshotLeftId` = `' + name + '`.`' + left + '` and `snapshotRightId` = `' + name + '`.`' + right + '`' + ) + expect(query.sql).toContain(`/*+ JOIN_FIXED_ORDER() JOIN_INDEX(${name}) */`) + expect(query.sql).toContain('`snapshotMembership` between ? and ?') + expect(query.bindings).toEqual([tableId, 41, 1, 3]) + const legacy = walletSnapshotSourceQuery(k, table, 41).toSQL() + expect(legacy.sql).not.toContain('snapshot_relation_keys') + expect(legacy.sql).toContain('or exists') + } +) + +test.each([6, 64])( + 'MySQL metadata resolves DATABASE() without a configured connection database and caches %s columns', + async count => { + const { source, db, queries } = fixture(count) + try { + const view = await source.openWalletReadSnapshot(identity) + const first = await view.readPage('txLabels') + expect(first.rows[0]).toMatchObject({ + txLabelId: 1, + userId: 1, + label: 'retained', + isDeleted: false, + created_at: when + }) + expect(first.payloadBytes).toBe(524) + expect(first.done).toBe(true) + expect(await view.readPage('txLabels')).toEqual(first) + expect(queries.filter(query => query.sql.startsWith('SELECT COLUMN_NAME'))).toHaveLength(1) + await view.close() + } finally { + await source.destroy() + await db.destroy() + } + } +) + +test.each([0, 65])('rejects an unsupported schema with %s columns before row payload queries', async count => { + const { source, queries } = fixture(count) + try { + const view = await source.openWalletReadSnapshot(identity) + await expect(view.readPage('txLabels')).rejects.toThrow('Unsupported snapshot schema') + await expect(view.closed).rejects.toThrow('Unsupported snapshot schema') + expect(queries.some(query => query.sql.includes('__snapshotBytes'))).toBe(false) + } finally { + await source.destroy() + } +}) + +test.each([-1, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1])( + 'invalid driver row charge %s cannot admit payload allocation', + async bytes => { + const { source, queries } = fixture(6, bytes) + try { + const view = await source.openWalletReadSnapshot(identity) + await expect(view.readPage('txLabels')).rejects.toThrow('Invalid snapshot row size') + await expect(view.closed).rejects.toThrow('Invalid snapshot row size') + expect(queries.some(query => query.sql.includes('`tx_labels`.*'))).toBe(false) + } finally { + await source.destroy() + } + } +) + +test('MySQL certificate pages preserve the collation-aware auxiliary key and indexed source lookup', () => { + const k = knex({ client: 'mysql2' }) + const q = walletSnapshotSourceQuery(k, 'certificateFields', 41, true, true, true) + .select('certificate_fields.*') + .toSQL() + expect(q.sql).toContain('/*+ JOIN_FIXED_ORDER() JOIN_INDEX(certificate_fields) */') + expect(q.sql).toContain( + 'from `snapshot_certificate_field_keys` FORCE INDEX (`PRIMARY`) cross join `certificate_fields`' + ) + expect(q.sql).toContain( + '`snapshotFieldName` = `certificate_fields`.`fieldName` and `snapshotCertificateId` = `certificate_fields`.`certificateId`' + ) + expect(q.sql).toContain('`snapshotMembership` between ? and ?') + expect(q.bindings).toEqual([41, 1, 3]) + const legacy = walletSnapshotSourceQuery(k, 'certificateFields', 41, true, true).toSQL() + expect(legacy.sql).not.toContain('snapshot_certificate_field_keys') + expect(legacy.sql).toContain('or exists') +}) + +test.each([ + ['provenTxReqs', 'proven_tx_reqs', 'provenTxReqId', 0], + ['provenTxs', 'proven_txs', 'provenTxId', 1] +] as const)( + 'MySQL %s pages retain the profile range plan and indexed source join', + async (table, name, key, tableId) => { + const k = knex({ client: 'mysql2' }) + try { + const query = walletSnapshotSourceQuery(k, table, 41, true, true, true, true) + .select(`${name}.*`) + .where('rowId', '>', 700) + .orderBy('rowId') + .limit(32) + .toSQL() + expect(query.sql).toContain(`/*+ JOIN_FIXED_ORDER() JOIN_INDEX(${name}) */`) + expect(query.sql).toContain( + 'from `snapshot_global_keys` FORCE INDEX (`snapshot_global_page`) cross join `' + name + '`' + ) + expect(query.sql).toContain('`rowId` = `' + name + '`.`' + key + '`') + expect(query.sql).toContain('`tableId` = ? and `userId` = ? and `present` = ? and `refs` > ? and `rowId` > ?') + expect(query.sql).toContain('order by `rowId` asc limit ?') + expect(query.bindings).toEqual([tableId, 41, 1, 0, 700, 32]) + const legacy = walletSnapshotSourceQuery(k, table, 41).toSQL() + expect(legacy.sql).not.toContain('snapshot_global_keys') + expect(legacy.sql).not.toContain('JOIN_FIXED_ORDER') + expect(legacy.sql).toContain('exists') + } finally { + await k.destroy() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.test.ts new file mode 100644 index 000000000..37166365a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.test.ts @@ -0,0 +1,707 @@ +import { WERR_NOT_IMPLEMENTED } from '../../sdk/WERR_errors' +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' +import { + removeSnapshotCertificateIndexes, + SNAPSHOT_CERTIFICATE_INDEX_MIGRATION +} from '../schema/snapshotCertificateIndexMigration' +import { removeSnapshotProfileIndexes, SNAPSHOT_PROFILE_INDEX_MIGRATION } from '../schema/snapshotProfileIndexMigration' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageIdb } from '../StorageIdb' +import { StorageProvider } from '../StorageProvider' +import type { + WalletSnapshotCursor, + WalletSnapshotTable, + WalletReadSnapshot, + PackedSnapshotRow, + WalletSnapshotTables +} from './WalletReadSnapshot' +import { runInSeries } from '../../utility/runInSeries' +import { createKnexWalletSnapshotPageReader, walletSnapshotSourceQuery } from './KnexWalletReadSnapshot' +import { + removeSnapshotRelationIndexes, + SNAPSHOT_RELATION_INDEX_MIGRATION +} from '../schema/snapshotRelationIndexMigration' + +const identity = '02' + '11'.repeat(32) +const foreignIdentity = '03' + '22'.repeat(32) +const date = '2026-01-01T00:00:00.000Z' +const timestamp = { created_at: date, updated_at: date } +const stores: StorageKnex[] = [] +const directories: string[] = [] + +async function fixture( + historicalIndexes = false +): Promise<{ source: StorageKnex; writer: StorageKnex; userId: number; otherId: number }> { + const directory = await mkdtemp(join(tmpdir(), 'wallet-keyset-')) + directories.push(directory) + const open = () => { + const store = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 1000 + }) + }) + stores.push(store) + return store + } + const source = open() + await source.knex.raw('PRAGMA journal_mode = WAL') + if (historicalIndexes) await migrateBeforeSqliteGeneration(source, 'keyset source', 'source-storage') + else await source.migrate('keyset source', 'source-storage') + await source.makeAvailable() + const { user } = await source.findOrInsertUser(identity) + const { user: other } = await source.findOrInsertUser(foreignIdentity) + const writer = open() + await writer.makeAvailable() + return { source, writer, userId: user.userId, otherId: other.userId } +} + +async function labels(source: StorageKnex, userId: number, count: number): Promise { + await runInSeries( + Array.from({ length: count }, (_, index) => index), + async index => { + await source.knex('tx_labels').insert({ + ...timestamp, + userId, + label: `label-${index}`, + isDeleted: index % 3 === 0 + }) + } + ) +} + +async function all( + view: WalletReadSnapshot, + table: T, + expectedMaximumRows: number +): Promise>> { + const rows: Array> = [] + const positions = new Set() + let cursor: WalletSnapshotCursor | undefined + for (;;) { + const page = await view.readPage(table, cursor, { maxRows: 1 }) + expect(page.rows.length).toBeLessThanOrEqual(1) + rows.push(...page.rows) + expect(rows.length).toBeLessThanOrEqual(expectedMaximumRows) + if (page.done) return rows + expect(page.rows).toHaveLength(1) + expect(page.cursor).toBeDefined() + const position = JSON.stringify(page.cursor!.after) + expect(positions.has(position)).toBe(false) + positions.add(position) + cursor = page.cursor + } +} + +afterEach(async () => { + jest.restoreAllMocks() + await runInSeries(stores.splice(0), source => source.destroy()) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('omitted auxiliary-index arguments preserve legacy profile queries and retained pages before migration', async () => { + const { source, userId, otherId } = await fixture() + await removeSnapshotProfileIndexes(source.knex) + await source.knex('knex_migrations').where('name', SNAPSHOT_PROFILE_INDEX_MIGRATION).delete() + await removeSnapshotRelationIndexes(source.knex) + await source.knex('knex_migrations').where('name', SNAPSHOT_RELATION_INDEX_MIGRATION).delete() + await labels(source, userId, 2) + await labels(source, otherId, 2) + const expected = await source.findTxLabels({ partial: { userId } }) + const view = await source.openReadSnapshot() + try { + const selected = await view.read(trx => + walletSnapshotSourceQuery(source.toDb(trx), 'txLabels', userId).select('txLabelId').orderBy('txLabelId') + ) + expect(selected).toEqual(expected.map(({ txLabelId }) => ({ txLabelId }))) + const reader = createKnexWalletSnapshotPageReader(source, userId, '12'.repeat(32), view) + const page = await reader('txLabels', undefined, { maxRows: 10 }) + expect(page.rows).toEqual(expected) + expect(page.done).toBe(true) + expect(page.cursor?.after).toEqual([expected.at(-1)!.txLabelId]) + expect( + await view.read(trx => walletSnapshotSourceQuery(source.toDb(trx), 'txLabelMaps', userId).select('*')) + ).toEqual([]) + expect((await reader('txLabelMaps')).rows).toEqual([]) + } finally { + await view.close() + } +}) + +test('keyset pages pin profile, source metadata, equal timestamps and tombstones across independent writes', async () => { + const { source, writer, userId, otherId } = await fixture() + await labels(source, userId, 9) + await labels(source, otherId, 9) + const expected = await source.findTxLabels({ partial: { userId } }) + await writer.knex('settings').update({ storageName: 'uncached name' }) + expect(source.getSettings().storageName).toBe('keyset source') + const view = await source.openWalletReadSnapshot(identity) + expect(view.version).toBe(1) + expect(view.snapshotId).toMatch(/^[a-f0-9]{64}$/) + expect(view.sourceStorage.storageName).toBe('uncached name') + expect(view.user.userId).toBe(userId) + expect(view.isOpen).toBe(true) + await writer.knex('tx_labels').where({ txLabelId: expected[0].txLabelId }).update({ label: 'new value' }) + await writer.knex('tx_labels').where({ txLabelId: expected[4].txLabelId }).del() + await writer.findOrInsertTxLabel(userId, 'new row') + await writer.knex('users').where({ userId }).update({ activeStorage: 'new primary' }) + const first = await view.readPage('txLabels', undefined, { maxRows: 3 }) + expect(first.rows).toEqual(expected.slice(0, 3)) + expect(first.done).toBe(false) + expect(await view.readPage('txLabels', undefined, { maxRows: 3 })).toEqual(first) + // Returned metadata is descriptive; mutating it cannot change the bound query. + view.user.userId = otherId + const next = await view.readPage('txLabels', first.cursor, { maxRows: 3 }) + expect(next.rows).toEqual(expected.slice(3, 6)) + expect(await all(view, 'txLabels', expected.length)).toEqual(expected) + await view.close() + await view.closed + expect(view.isOpen).toBe(false) + await expect(view.readPage('txLabels')).rejects.toThrow('closed') + const fresh = await source.openWalletReadSnapshot(identity) + expect(fresh.user.activeStorage).toBe('new primary') + await expect(fresh.readPage('txLabels', first.cursor)).rejects.toThrow('cursor') + expect(await all(fresh, 'txLabels', expected.length)).not.toEqual(expected) + await fresh.close() +}) + +test('byte preflight stops before an oversized row without loading its payload or skipping it', async () => { + const { source, userId } = await fixture() + await labels(source, userId, 2) + await source.knex('tx_labels').insert({ ...timestamp, userId, label: 'x'.repeat(500000), isDeleted: false }) + const view = await source.openWalletReadSnapshot(identity) + const first = await view.readPage('txLabels', undefined, { maxRows: 1 }) + const second = await view.readPage('txLabels', first.cursor, { maxRows: 1, maxBytes: first.payloadBytes }) + expect(second.rows).toHaveLength(1) + const bounded = await view.readPage('txLabels', undefined, { maxRows: 3, maxBytes: first.payloadBytes * 2 }) + expect(bounded.rows).toHaveLength(2) + expect(bounded.payloadBytes).toBe(first.payloadBytes * 2) + expect(bounded.done).toBe(false) + const queries: string[] = [] + source.knex.on('query', q => queries.push(q.sql)) + await expect(view.readPage('txLabels', second.cursor)).rejects.toThrow('row exceeds maxBytes') + await expect(view.closed).rejects.toThrow('row exceeds maxBytes') + expect(queries.filter(sql => /select/i.test(sql))).toHaveLength(1) + expect(queries[0]).toContain('length(cast(') + expect(queries[0]).not.toContain('`tx_labels`.*') + expect(queries[0]).not.toMatch(/offset|count\(/i) + const next = await source.openWalletReadSnapshot(identity) + expect((await next.readPage('txLabels', undefined, { maxRows: 3, maxBytes: 2000000 })).rows).toHaveLength(3) + await next.close() +}) + +test.each([ + { maxRows: 0 }, + { maxRows: -1 }, + { maxRows: 1.5 }, + { maxRows: 1001 }, + { maxRows: NaN }, + { maxBytes: 0 }, + { maxBytes: -1 }, + { maxBytes: 1.5 }, + { maxBytes: 16777217 }, + { maxBytes: Infinity } +])('rejects invalid allocation limits before executing SQL: %j', async limits => { + const { source } = await fixture() + const view = await source.openWalletReadSnapshot(identity) + const read = jest.spyOn(source, 'toDb') + await expect(view.readPage('txLabels', undefined, limits)).rejects.toThrow('integer') + expect(read).not.toHaveBeenCalled() + expect(view.isOpen).toBe(true) + await view.close() +}) + +test('rejects table and cursor confusion before querying and detaches caller cursors', async () => { + const { source, userId } = await fixture() + await labels(source, userId, 3) + const view = await source.openWalletReadSnapshot(identity) + const { cursor } = await view.readPage('txLabels', undefined, { maxRows: 1 }) + const read = jest.spyOn(source, 'toDb') + for (const table of ['settings', 'users', '__proto__', 'constructor', 'tx_labels']) { + await expect(view.readPage(table as WalletSnapshotTable)).rejects.toThrow('table') + } + for (const invalid of [ + null, + { ...cursor, version: 2 }, + { ...cursor, snapshotId: 'different' }, + { ...cursor, table: 'outputs' }, + { ...cursor, after: [] }, + { ...cursor, after: [0] }, + { ...cursor, after: [-1] }, + { ...cursor, after: [1.1] }, + { ...cursor, after: ['1'] }, + { ...cursor, after: [Number.MAX_SAFE_INTEGER + 1] }, + { ...cursor, after: [1, 2] }, + { ...cursor, after: {} } + ]) + await expect(view.readPage('txLabels', invalid as WalletSnapshotCursor)).rejects.toThrow('cursor') + expect(read).not.toHaveBeenCalled() + const mutable = { ...cursor!, after: [...cursor!.after] } + const pending = view.readPage('txLabels', mutable) + mutable.after[0] = 99999 + expect((await pending).rows).toHaveLength(2) + await view.close() +}) + +test('unsupported providers and invalid identities refuse explicitly and failed opening releases its slot', async () => { + const idb = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + expect(idb.supportsWalletReadSnapshot()).toBe(false) + await expect(idb.openWalletReadSnapshot(identity)).rejects.toThrow('not supported') + const { source } = await fixture() + expect(source.supportsWalletReadSnapshot()).toBe(true) + const read = jest.spyOn(source, 'openReadSnapshot') + for (const key of [ + '', + '04' + '11'.repeat(32), + '02' + 'gg'.repeat(32), + 'x' + identity, + identity + 'x', + { toString: () => identity } as unknown as string + ]) { + await expect(source.openWalletReadSnapshot(key)).rejects.toThrow('identityKey') + } + expect(read).not.toHaveBeenCalled() + await expect(source.openWalletReadSnapshot('02' + '33'.repeat(32))).rejects.toThrow('existing wallet profile') + const view = await source.openWalletReadSnapshot(identity) + await view.close() + jest.spyOn(source, 'supportsRetainedReadSnapshot').mockReturnValue(false) + expect(source.supportsWalletReadSnapshot()).toBe(false) + await expect(source.openWalletReadSnapshot(identity)).rejects.toThrow('not supported') +}) + +async function seedClosure(source: StorageKnex, userId: number, otherId: number): Promise { + const k = source.knex + await k('output_baskets').del() + for (const id of [1, 2, 3]) { + await k('proven_txs').insert({ + ...timestamp, + provenTxId: id, + txid: String(id).repeat(64), + height: id, + index: 0, + merklePath: Buffer.from([id, 0, 255]), + rawTx: Buffer.from([id, 1, 255]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + await k('transactions').insert({ + ...timestamp, + transactionId: id, + userId: id === 2 ? otherId : userId, + provenTxId: id === 3 ? null : id, + status: 'completed', + reference: `tx-${id}`, + isOutgoing: true, + satoshis: 0, + description: `tx-${id}`, + txid: String(id).repeat(64), + rawTx: Buffer.from([id, 2, 255]), + inputBEEF: Buffer.from([id, 3, 255]) + }) + await k('proven_tx_reqs').insert({ + ...timestamp, + provenTxReqId: id, + provenTxId: id, + txid: String(id).repeat(64), + status: 'completed', + attempts: 0, + notified: true, + history: '{}', + notify: '{}', + rawTx: Buffer.from([id, 4, 255]), + wasBroadcast: true + }) + await k('output_baskets').insert({ + ...timestamp, + basketId: id, + userId: id === 2 ? otherId : userId, + name: `basket-${id}`, + isDeleted: id === 3 + }) + await k('outputs').insert({ + ...timestamp, + outputId: id, + userId: id === 2 ? otherId : userId, + transactionId: id, + basketId: id, + spendable: false, + change: true, + vout: 0, + satoshis: 1, + providedBy: 'you', + purpose: '', + type: 'P2PKH', + lockingScript: Buffer.from([id, 5, 255]) + }) + await k('commissions').insert({ + ...timestamp, + commissionId: id, + userId: id === 2 ? otherId : userId, + transactionId: id, + satoshis: 0, + keyOffset: 'offset', + isRedeemed: true, + lockingScript: Buffer.from([id, 6, 255]) + }) + await k('output_tags').insert({ + ...timestamp, + outputTagId: id, + userId: id === 2 ? otherId : userId, + tag: `tag-${id}`, + isDeleted: id === 3 + }) + await k('output_tags_map').insert({ ...timestamp, outputTagId: id, outputId: id, isDeleted: id === 3 }) + await k('tx_labels').insert({ + ...timestamp, + txLabelId: id, + userId: id === 2 ? otherId : userId, + label: `label-${id}`, + isDeleted: id === 3 + }) + await k('tx_labels_map').insert({ ...timestamp, txLabelId: id, transactionId: id, isDeleted: id === 3 }) + await k('certificates').insert({ + ...timestamp, + certificateId: id, + userId: id === 2 ? otherId : userId, + serialNumber: `serial-${id}`, + type: 'type', + certifier: identity, + subject: identity, + revocationOutpoint: 'a'.repeat(64) + '.0', + signature: 'signature', + isDeleted: id === 3 + }) + for (const fieldName of ['a', 'Z', 'é', '😀']) + await k('certificate_fields').insert({ + ...timestamp, + certificateId: id, + userId: id === 2 ? otherId : userId, + fieldName, + fieldValue: `value-${id}`, + masterKey: 'key' + }) + await k('sync_states').insert({ + ...timestamp, + syncStateId: id, + userId: id === 2 ? otherId : userId, + storageIdentityKey: `peer-${id}`, + storageName: `peer-${id}`, + status: 'unknown', + init: true, + refNum: `state-${id}`, + syncMap: '{}', + when: date + }) + } + // Composite positions must handle repeated first keys and preserve deleted mappings. + await k('output_tags_map').insert({ ...timestamp, outputTagId: 1, outputId: 3, isDeleted: true }) + await k('tx_labels_map').insert({ ...timestamp, txLabelId: 1, transactionId: 3, isDeleted: true }) +} + +test('all thirteen tables retain original rows, packed binary and composite key order without foreign profiles', async () => { + const { source, userId, otherId } = await fixture() + await seedClosure(source, userId, otherId) + const view = await source.openWalletReadSnapshot(identity) + const cases: Array<[WalletSnapshotTable, string, number[]]> = [ + ['provenTxs', 'provenTxId', [1, 3]], + ['provenTxReqs', 'provenTxReqId', [1, 3]], + ['outputBaskets', 'basketId', [1, 3]], + ['transactions', 'transactionId', [1, 3]], + ['outputs', 'outputId', [1, 3]], + ['commissions', 'commissionId', [1, 3]], + ['outputTags', 'outputTagId', [1, 3]], + ['txLabels', 'txLabelId', [1, 3]], + ['certificates', 'certificateId', [1, 3]], + ['syncStates', 'syncStateId', [1, 3]] + ] + for (const [table, key, ids] of cases) { + const rows = (await all(view, table, ids.length)) as unknown as Array> + expect(rows.map(row => row[key])).toEqual(ids) + for (const row of rows) { + expect(row.created_at).toEqual(new Date(date)) + expect(row.updated_at).toEqual(new Date(date)) + if ('userId' in row) expect(row.userId).toBe(userId) + for (const flag of [ + 'isDeleted', + 'isOutgoing', + 'isRedeemed', + 'spendable', + 'change', + 'notified', + 'wasBroadcast', + 'init' + ]) { + if (flag in row) expect(typeof row[flag]).toBe('boolean') + } + for (const binary of ['rawTx', 'inputBEEF', 'merklePath', 'lockingScript']) { + if (row[binary] !== undefined) { + expect(row[binary]).toBeInstanceOf(Uint8Array) + expect(Array.isArray(row[binary])).toBe(false) + expect((row[binary] as Uint8Array)[2]).toBe(255) + expect((row[binary] as Uint8Array).buffer.byteLength).toBe((row[binary] as Uint8Array).byteLength) + } + } + } + } + expect((await all(view, 'txLabelMaps', 3)).map(row => [row.txLabelId, row.transactionId, row.isDeleted])).toEqual([ + [1, 1, false], + [1, 3, true], + [3, 3, true] + ]) + expect((await all(view, 'outputTagMaps', 3)).map(row => [row.outputTagId, row.outputId, row.isDeleted])).toEqual([ + [1, 1, false], + [1, 3, true], + [3, 3, true] + ]) + expect((await all(view, 'certificateFields', 8)).map(row => [row.fieldName, row.certificateId])).toEqual( + ['Z', 'a', 'é', '😀'].flatMap(field => [ + [field, 1], + [field, 3] + ]) + ) + const states = await view.readPage('syncStates') + expect(states.rows[0].init).toBe(true) + expect(states.rows[0].when).toEqual(new Date(date)) + expect(states.rows[0].syncMap).toBe('{}') + const proof = await view.readPage('provenTxs') + proof.rows[0].rawTx[0] = 99 + expect((await view.readPage('provenTxs')).rows[0].rawTx[0]).toBe(1) + await view.close() +}) + +test.each(['txLabelMaps', 'outputTagMaps', 'certificateFields'] as const)( + 'inconsistent owned relationships reject instead of leaking or silently dropping a row: %s', + async table => { + const { source, userId, otherId } = await fixture() + await seedClosure(source, userId, otherId) + if (table === 'txLabelMaps') + await source.knex('tx_labels_map').insert({ ...timestamp, txLabelId: 1, transactionId: 2 }) + if (table === 'outputTagMaps') + await source.knex('output_tags_map').insert({ ...timestamp, outputTagId: 1, outputId: 2 }) + if (table === 'certificateFields') + await source.knex('certificate_fields').where({ certificateId: 2 }).update({ userId }) + const view = await source.openWalletReadSnapshot(identity) + await expect(view.readPage(table)).rejects.toThrow('does not belong') + await expect(view.closed).rejects.toThrow('does not belong') + } +) + +test('certificate keys preserve empty, embedded-NUL and 100-code-point names with bounded cursor validation', async () => { + const { source, userId, otherId } = await fixture() + await seedClosure(source, userId, otherId) + const names = ['', 'a\0x', 'a\0y', '😀'.repeat(100)] + for (const fieldName of names) + await source.knex('certificate_fields').insert({ + ...timestamp, + userId, + certificateId: 1, + fieldName, + fieldValue: 'value', + masterKey: 'key' + }) + const view = await source.openWalletReadSnapshot(identity) + const rows = await all(view, 'certificateFields', 12) + for (const name of names) expect(rows.some(row => row.fieldName === name)).toBe(true) + expect(rows).toHaveLength(12) + for (const fieldName of [100, 'a'.repeat(101), '😀'.repeat(101)]) { + await expect( + view.readPage('certificateFields', { + version: 1, + snapshotId: view.snapshotId, + table: 'certificateFields', + after: [fieldName, 1] + }) + ).rejects.toThrow('cursor') + } + await view.close() +}) + +test.each(['a'.repeat(101), 'a'.repeat(401)])( + 'oversized stored certificate key rejects without loading its row (%#)', + async fieldName => { + const { source, userId, otherId } = await fixture() + await seedClosure(source, userId, otherId) + await source.knex('certificate_fields').del() + await source + .knex('certificate_fields') + .insert({ ...timestamp, userId, certificateId: 1, fieldName, fieldValue: 'x'.repeat(500000), masterKey: 'key' }) + const view = await source.openWalletReadSnapshot(identity) + const queries: string[] = [] + source.knex.on('query', q => queries.push(q.sql)) + await expect(view.readPage('certificateFields', undefined, { maxBytes: 16777216 })).rejects.toThrow('cursor') + await expect(view.closed).rejects.toThrow('cursor') + expect(queries.some(sql => sql.includes('`certificate_fields`.*'))).toBe(false) + } +) + +test('empty tables, maximum limits and caller cancellation preserve cleanup semantics', async () => { + const { source } = await fixture() + const controller = new AbortController() + const view = await source.openWalletReadSnapshot(identity, { signal: controller.signal }) + expect(await view.readPage('txLabels', undefined, { maxRows: 1000, maxBytes: 16777216 })).toEqual({ + rows: [], + payloadBytes: 0, + done: true, + cursor: undefined + }) + controller.abort() + await expect(view.readPage('txLabels')).rejects.toThrow('cancelled') + await view.closed + const fresh = await source.openWalletReadSnapshot(identity) + await fresh.close() +}) + +test('concurrent reads refuse instead of queueing and expiry invalidates cursors', async () => { + const { source, userId } = await fixture() + await labels(source, userId, 3) + const view = await source.openWalletReadSnapshot(identity, { lifetimeMs: 10000 }) + const reading = view.readPage('txLabels', undefined, { maxRows: 1 }) + await expect(view.readPage('txLabels')).rejects.toThrow('read in flight') + const first = await reading + jest.spyOn(Date, 'now').mockReturnValue(view.expiresAt) + await expect(view.readPage('txLabels', first.cursor)).rejects.toThrow('expired') + await view.closed +}) + +test.each([ + [false, false, false], + [false, false, true], + [false, true, false], + [false, true, true], + [true, false, false], + [true, false, true], + [true, true, false], + [true, true, true] +])( + 'SQL keys support forward seeks without OFFSET or counts (profile=%s, relation=%s, certificate=%s)', + async (profileIndexes, relationIndexes, certificateIndexes) => { + const { source, userId, otherId } = await fixture(true) + if (!profileIndexes) { + await removeSnapshotProfileIndexes(source.knex) + await source.knex('knex_migrations').where('name', SNAPSHOT_PROFILE_INDEX_MIGRATION).delete() + } + if (!relationIndexes) { + await removeSnapshotRelationIndexes(source.knex) + await source.knex('knex_migrations').where('name', SNAPSHOT_RELATION_INDEX_MIGRATION).delete() + } + if (!certificateIndexes) { + await removeSnapshotCertificateIndexes(source.knex) + await source.knex('knex_migrations').where('name', SNAPSHOT_CERTIFICATE_INDEX_MIGRATION).delete() + } + await seedClosure(source, userId, otherId) + const view = await source.openWalletReadSnapshot(identity) + const requests: Array<{ sql: string; bindings: Knex.RawBinding[] }> = [] + const collect = (q: { sql: string; bindings: Knex.RawBinding[] }) => { + if (q.sql.startsWith('select')) requests.push(q) + } + source.knex.on('query', collect) + for (const table of ['txLabels', 'txLabelMaps', 'certificateFields'] as const) { + const first = await view.readPage(table, undefined, { maxRows: 1 }) + await view.readPage(table, first.cursor, { maxRows: 1 }) + } + source.knex.removeListener('query', collect) + await view.close() + const subsequent = requests.filter(q => q.sql.includes(' > ')) + expect(subsequent).toHaveLength(6) + expect(subsequent.filter(query => query.sql.includes('snapshot_profile_keys'))).toHaveLength(profileIndexes ? 2 : 0) + expect(subsequent.filter(query => query.sql.includes('snapshot_relation_keys'))).toHaveLength( + relationIndexes ? 2 : 0 + ) + expect(subsequent.filter(query => query.sql.includes('snapshot_certificate_field_keys'))).toHaveLength( + certificateIndexes ? 2 : 0 + ) + for (const query of subsequent) { + expect(query.sql).not.toMatch(/offset|count\(/i) + const plan = await source.knex.raw('EXPLAIN QUERY PLAN ' + query.sql, query.bindings) + const details = plan.map((row: { detail: string }) => row.detail).join('\n') + if (query.sql.includes('snapshot_profile_keys')) { + expect(details).toMatch( + /SEARCH snapshot_profile_keys USING COVERING INDEX .*\(snapshotTableId=\? AND snapshotUserId=\? AND snapshotRowId>\?(?: AND snapshotRowId<\?)?\)/ + ) + expect(details).toMatch(/SEARCH tx_labels USING INTEGER PRIMARY KEY \(rowid=\?\)/) + expect(details).not.toMatch(/SCAN |TEMP B-TREE/) + } else if (query.sql.includes('snapshot_relation_keys')) { + expect(details).toMatch( + /SEARCH snapshot_relation_keys USING COVERING INDEX .*snapshotTableId=\? AND snapshotUserId=\? AND \(snapshotLeftId,snapshotRightId\)>\(\?,\?\)/ + ) + expect(details).not.toMatch(/SCAN |TEMP B-TREE/) + } else if (query.sql.includes('snapshot_certificate_field_keys')) { + expect(details).toMatch( + /SEARCH snapshot_certificate_field_keys USING COVERING INDEX .*snapshotUserId=\? AND \(snapshotFieldName,snapshotCertificateId\)>\(\?,\?\)/ + ) + expect(details).toMatch(/SEARCH certificate_fields USING INDEX .*fieldName=\? AND certificateId=\?/) + expect(details).not.toMatch(/SCAN |TEMP B-TREE/) + } else { + expect(details).toMatch(/SEARCH (tx_labels|tx_labels_map|certificate_fields) USING .*\(.*>\(?\?/) + } + } + } +) + +test('an owned certificate with a foreign-user field rejects without returning that field', async () => { + const { source, userId, otherId } = await fixture() + await seedClosure(source, userId, otherId) + await source.knex('certificate_fields').where({ certificateId: 1, fieldName: 'a' }).update({ userId: otherId }) + const view = await source.openWalletReadSnapshot(identity) + await expect(view.readPage('certificateFields')).rejects.toThrow('does not belong') + await expect(view.closed).rejects.toThrow('does not belong') +}) + +test.each(['txLabelMaps', 'outputTagMaps'] as const)( + 'a foreign label/tag linked to an owned parent also rejects: %s', + async table => { + const { source, userId, otherId } = await fixture() + await seedClosure(source, userId, otherId) + if (table === 'txLabelMaps') + await source.knex('tx_labels_map').insert({ ...timestamp, txLabelId: 2, transactionId: 1 }) + else await source.knex('output_tags_map').insert({ ...timestamp, outputTagId: 2, outputId: 1 }) + const view = await source.openWalletReadSnapshot(identity) + await expect(view.readPage(table)).rejects.toThrow('does not belong') + await expect(view.closed).rejects.toThrow('does not belong') + } +) + +test('a page query failure is observed through read and cleanup before the provider can open a fresh view', async () => { + const { source, userId } = await fixture() + await labels(source, userId, 1) + const failure = new Error('Synthetic page query failure') + const rejectPage = (query: { sql: string }): void => { + if (query.sql.includes('__snapshotBytes')) throw failure + } + source.knex.on('query', rejectPage) + const view = await source.openWalletReadSnapshot(identity) + await expect(view.readPage('txLabels')).rejects.toBe(failure) + await expect(view.closed).rejects.toBe(failure) + expect(view.isOpen).toBe(false) + source.knex.removeListener('query', rejectPage) + const fresh = await source.openWalletReadSnapshot(identity) + expect((await fresh.readPage('txLabels')).rows).toHaveLength(1) + await fresh.close() +}) + +test('a Knex provider that disables packed snapshots refuses before opening or querying', async () => { + const k = knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + try { + jest.spyOn(source, 'supportsWalletReadSnapshot').mockReturnValue(false) + const available = jest + .spyOn(source, 'makeAvailable') + .mockRejectedValue(new Error('unexpected database acquisition')) + const open = jest.spyOn(source, 'openReadSnapshot').mockRejectedValue(new Error('unexpected snapshot acquisition')) + const request = source.openWalletReadSnapshot('02' + '11'.repeat(32)) + await expect(request).rejects.toBeInstanceOf(WERR_NOT_IMPLEMENTED) + await expect(request).rejects.toThrow('Wallet read snapshot pages are not supported by this provider') + expect(available).not.toHaveBeenCalled() + expect(open).not.toHaveBeenCalled() + } finally { + await source.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.ts new file mode 100644 index 000000000..0138ebd9e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/KnexWalletReadSnapshot.ts @@ -0,0 +1,554 @@ +import { readGenerationIndexState, type SnapshotIndexState } from '../schema/snapshotSqliteIndexState' +import { readSnapshotGlobalIndexState } from '../schema/snapshotGlobalIndexMigration' +import { readSnapshotCertificateIndexState } from '../schema/snapshotCertificateIndexMigration' +import { readSnapshotRelationIndexState } from '../schema/snapshotRelationIndexMigration' +import { readSnapshotProfileIndexState } from '../schema/snapshotProfileIndexMigration' +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import { Random, Utils } from '@bsv/sdk' +import type { Knex } from 'knex' +import type { StorageKnex } from '../StorageKnex' +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' +import { WERR_INVALID_PARAMETER, WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { RetainedReadSnapshot } from './RetainedReadSnapshot' +import type { + PackedSnapshotRow, + WalletReadSnapshot, + WalletReadSnapshotOptions, + WalletSnapshotCursor, + WalletSnapshotPage, + WalletSnapshotPageLimits, + WalletSnapshotTable, + WalletSnapshotTables +} from './WalletReadSnapshot' + +interface TableDefinition { + name: string + keys: string[] + booleans?: string[] + dates?: string[] +} + +// Preserve existing logical keys and the standard-table indexes used by legacy +// OFFSET checkpoints. Auxiliary profile keys retain the same snapshot order. Map/field order is explicitly +// storage-key order, not the canonical BRC-38 array order. +const definitions: Record = { + provenTxs: { name: 'proven_txs', keys: ['provenTxId'] }, + provenTxReqs: { + name: 'proven_tx_reqs', + keys: ['provenTxReqId'], + booleans: ['notified', 'wasBroadcast'] + }, + outputBaskets: { name: 'output_baskets', keys: ['basketId'], booleans: ['isDeleted'] }, + transactions: { name: 'transactions', keys: ['transactionId'], booleans: ['isOutgoing'] }, + commissions: { name: 'commissions', keys: ['commissionId'], booleans: ['isRedeemed'] }, + outputs: { name: 'outputs', keys: ['outputId'], booleans: ['spendable', 'change'] }, + outputTags: { name: 'output_tags', keys: ['outputTagId'], booleans: ['isDeleted'] }, + outputTagMaps: { + name: 'output_tags_map', + keys: ['outputTagId', 'outputId'], + booleans: ['isDeleted'] + }, + txLabels: { name: 'tx_labels', keys: ['txLabelId'], booleans: ['isDeleted'] }, + txLabelMaps: { + name: 'tx_labels_map', + keys: ['txLabelId', 'transactionId'], + booleans: ['isDeleted'] + }, + certificates: { name: 'certificates', keys: ['certificateId'], booleans: ['isDeleted'] }, + certificateFields: { name: 'certificate_fields', keys: ['fieldName', 'certificateId'] }, + syncStates: { name: 'sync_states', keys: ['syncStateId'], booleans: ['init'], dates: ['when'] } +} + +const auxiliaryTableIds: Partial> = { + transactions: 0, + outputs: 1, + certificates: 2, + txLabels: 3, + outputBaskets: 4, + outputTags: 5, + commissions: 6, + syncStates: 7 +} + +const auxiliaryRelationTableIds: Partial> = { + txLabelMaps: 0, + outputTagMaps: 1 +} + +function definition(table: WalletSnapshotTable): TableDefinition { + if (!Object.hasOwn(definitions, table)) throw new WERR_INVALID_PARAMETER('table', 'a wallet snapshot table') + return definitions[table] +} + +function bound(value: number | undefined, fallback: number, ceiling: number, name: string): number { + const result = value ?? fallback + if (!Number.isSafeInteger(result) || result < 1 || result > ceiling) { + throw new WERR_INVALID_PARAMETER(name, `an integer from 1 to ${ceiling}`) + } + return result +} + +function position( + cursor: WalletSnapshotCursor | undefined, + snapshotId: string, + table: WalletSnapshotTable, + keys: string[] +): Array | undefined { + if (cursor === undefined) return undefined + if ( + cursor?.version !== 1 || + cursor.snapshotId !== snapshotId || + cursor.table !== table || + !Array.isArray(cursor.after) || + cursor.after.length !== keys.length || + cursor.after.some((value, index) => + keys[index] === 'fieldName' + ? typeof value !== 'string' || value.length > 200 || Array.from(value).length > 100 + : !Number.isSafeInteger(value) || (value as number) < 1 + ) + ) { + throw new WERR_INVALID_PARAMETER('cursor', 'a version-one position for this snapshot and table') + } + // Detach before the asynchronous database boundary. + return [...cursor.after] +} + +/** Lexicographic seek, using the same database collation as ORDER BY. */ +function seek( + query: Knex.QueryBuilder, + keys: string[], + after: Array, + inclusive: boolean, + mysql: boolean +): void { + if (mysql && keys.length === 2) { + const first = inclusive ? '<' : '>' + const last = inclusive ? '<=' : '>' + void query.whereRaw(`(?? ${first} ? OR (?? = ? AND ?? ${last} ?))`, [ + keys[0], + after[0], + keys[0], + after[0], + keys[1], + after[1] + ]) + return + } + void query.whereRaw( + `(${keys.map(() => '??').join(', ')}) ${inclusive ? '<=' : '>'} (${keys.map(() => '?').join(', ')})`, + [...keys, ...after] + ) +} + +function owned(k: Knex, table: string, id: string, source: string, userId: number): Knex.QueryBuilder { + return k(table) + .select(k.raw('?', [1])) + .where(`${table}.userId`, userId) + .whereRaw('?? = ??', [`${table}.${id}`, source]) +} + +const globalTableIds: Partial> = { + provenTxReqs: 0, + provenTxs: 1 +} +function relationSourceQuery( + k: Knex, + table: WalletSnapshotTable, + userId: number, + relationId: number, + state: SnapshotIndexState +): Knex.QueryBuilder { + const { name } = definitions[table] + + const [left, right] = definitions[table].keys + // The maintenance index begins with the same profile prefix. MySQL can + // choose it and sort the complete profile before LIMIT; bind paging to the + // auxiliary primary key whose suffix is the unchanged cursor order. + const sqliteKeys = state === 'v2' ? 'snapshot_relation_keys_v2' : 'snapshot_relation_keys' + const relationKeys = String(k.client.config.client).includes('mysql') + ? k.raw('?? FORCE INDEX (??)', ['snapshot_relation_keys', 'PRIMARY']) + : sqliteKeys + const query = k(relationKeys) + .crossJoin(name, function () { + void this.on('snapshotLeftId', '=', `${name}.${left}`).andOn('snapshotRightId', '=', `${name}.${right}`) + }) + .where({ snapshotTableId: relationId, snapshotUserId: userId }) + // Read the recorded membership and keep source lookups indexed even before + // InnoDB has refreshed cardinality statistics after bootstrap or bulk writes. + if (String(k.client.config.client).includes('mysql')) + void query.whereBetween('snapshotMembership', [1, 3]).hintComment(['JOIN_FIXED_ORDER()', `JOIN_INDEX(${name})`]) + return query +} +function certificateSourceQuery(k: Knex, userId: number, state: SnapshotIndexState): Knex.QueryBuilder { + const name = 'certificate_fields' + + const mysql = String(k.client.config.client).includes('mysql') + const sqliteKeys = state === 'v2' ? 'snapshot_certificate_field_keys_v2' : 'snapshot_certificate_field_keys' + const keys = mysql ? k.raw('?? FORCE INDEX (??)', ['snapshot_certificate_field_keys', 'PRIMARY']) : sqliteKeys + const query = k(keys) + .crossJoin(name, function () { + void this.on('snapshotFieldName', '=', `${name}.fieldName`).andOn( + 'snapshotCertificateId', + '=', + `${name}.certificateId` + ) + }) + .where('snapshotUserId', userId) + if (mysql) + void query + .whereBetween('snapshotMembership', [1, 3]) + .hintComment(['JOIN_FIXED_ORDER()', 'JOIN_INDEX(certificate_fields)']) + return query +} +function globalSourceQuery( + k: Knex, + table: WalletSnapshotTable, + userId: number, + globalId: number, + state: SnapshotIndexState +): Knex.QueryBuilder { + const { name } = definitions[table] + + const mysql = String(k.client.config.client).includes('mysql') + const sqliteKeys = state === 'v2' ? 'snapshot_global_keys_v2' : 'snapshot_global_keys' + const keys = mysql ? k.raw('?? FORCE INDEX (??)', ['snapshot_global_keys', 'snapshot_global_page']) : sqliteKeys + const query = k(keys) + .crossJoin(name, 'rowId', `${name}.${definitions[table].keys[0]}`) + .where({ tableId: globalId, userId, present: 1 }) + .where('refs', '>', 0) + if (mysql) void query.hintComment(['JOIN_FIXED_ORDER()', `JOIN_INDEX(${name})`]) + return query +} + +/** Shared profile selection for local paging and archive closure checks. */ +export function walletSnapshotSourceQuery( + k: Knex, + table: WalletSnapshotTable, + userId: number, + profileIndexes: SnapshotIndexState = false, + relationIndexes: SnapshotIndexState = false, + certificateIndexes: SnapshotIndexState = false, + globalIndexes: SnapshotIndexState = false +): Knex.QueryBuilder { + const { name } = definitions[table] + const tableId = auxiliaryTableIds[table] + if (profileIndexes && tableId !== undefined) + return k(profileIndexes === 'v2' ? 'snapshot_profile_keys_v2' : 'snapshot_profile_keys') + .crossJoin(name, 'snapshotRowId', `${name}.${definitions[table].keys[0]}`) + .where({ snapshotTableId: tableId, snapshotUserId: userId }) + .where(`${name}.userId`, userId) + const relationId = auxiliaryRelationTableIds[table] + if (relationIndexes && relationId !== undefined) + return relationSourceQuery(k, table, userId, relationId, relationIndexes) + if (certificateIndexes && table === 'certificateFields') return certificateSourceQuery(k, userId, certificateIndexes) + const globalId = globalTableIds[table] + if (globalIndexes && globalId !== undefined) return globalSourceQuery(k, table, userId, globalId, globalIndexes) + const query = k(name) + if (table === 'provenTxReqs') { + return query.whereExists(owned(k, 'transactions', 'txid', `${name}.txid`, userId)) + } + if (table === 'provenTxs') { + // Include proofs referenced only by a request as well as transaction proofs. + return query.where(function () { + void this.whereExists(owned(k, 'transactions', 'provenTxId', `${name}.provenTxId`, userId)).orWhereExists( + k('proven_tx_reqs') + .select(k.raw('?', [1])) + .whereRaw('?? = ??', ['proven_tx_reqs.provenTxId', `${name}.provenTxId`]) + .whereExists(owned(k, 'transactions', 'txid', 'proven_tx_reqs.txid', userId)) + ) + }) + } + if (table === 'txLabelMaps') { + return query.where(function () { + void this.whereExists(owned(k, 'tx_labels', 'txLabelId', `${name}.txLabelId`, userId)).orWhereExists( + owned(k, 'transactions', 'transactionId', `${name}.transactionId`, userId) + ) + }) + } + if (table === 'outputTagMaps') { + return query.where(function () { + void this.whereExists(owned(k, 'output_tags', 'outputTagId', `${name}.outputTagId`, userId)).orWhereExists( + owned(k, 'outputs', 'outputId', `${name}.outputId`, userId) + ) + }) + } + if (table === 'certificateFields') { + return query.where(function () { + void this.where(`${name}.userId`, userId).orWhereExists( + owned(k, 'certificates', 'certificateId', `${name}.certificateId`, userId) + ) + }) + } + return query.where(`${name}.userId`, userId) +} + +function normalize(storage: StorageKnex, row: Record, schema: TableDefinition): T { + for (const key of Object.keys(row)) { + const value = row[key] + if (value === null) row[key] = undefined + else if (key === 'created_at' || key === 'updated_at' || schema.dates?.includes(key) === true) { + row[key] = storage.validateDate(value as Date) + } else if (schema.booleans?.includes(key) === true) row[key] = value !== 0 && value !== false + else if (value instanceof Uint8Array) row[key] = new Uint8Array(value) + } + return row as T +} + +function charge(k: Knex, columns: string[], mysql: boolean): Knex.Raw { + const length = mysql ? 'octet_length(??)' : 'length(cast(?? as blob))' + const cellCharge = `2 * coalesce(${length}, 0) + 64` + return k.raw(`(${columns.map(() => cellCharge).join(' + ')}) as ??`, [...columns, '__snapshotBytes']) +} + +function keyColumn(k: Knex, key: string, mysql: boolean): string | Knex.Raw { + if (key !== 'fieldName') return key + const length = mysql ? 'octet_length(??)' : 'length(cast(?? as blob))' + // Preserve all characters, including embedded NUL and astral code points. + // SQLite substr/length(text) stop at NUL. Bound bytes in SQL before loading + // this key, then validate its 100-character schema limit in the cursor parser. + return k.raw(`case when ${length} <= 400 then ?? end as ??`, [key, key, key]) +} + +function relationGuard(k: Knex, table: WalletSnapshotTable, userId: number): Knex.Raw { + const name = definitions[table].name + const relations: Partial>> = { + txLabelMaps: [ + ['tx_labels', 'txLabelId'], + ['transactions', 'transactionId'] + ], + outputTagMaps: [ + ['output_tags', 'outputTagId'], + ['outputs', 'outputId'] + ], + certificateFields: [ + ['certificates', 'certificateId'], + ['users', 'userId'] + ] + } + const relation = relations[table] + if (relation === undefined) return k.raw('1 as ??', ['__snapshotOwned']) + return k.raw(`(${relation.map(() => 'exists ?').join(' and ')}) as ??`, [ + ...relation.map(([target, id]) => owned(k, target, id, `${name}.${id}`, userId)), + '__snapshotOwned' + ]) +} + +async function columnNames(k: Knex, table: string, mysql: boolean): Promise { + if (!mysql) return Object.keys(await k(table).columnInfo()) + // Knex's columnInfo binds client.config.connection.database, which is absent + // with an externally supplied mysql2 pool. Inspect this retained connection's + // actual database instead, with a bounded metadata result. + const [rows]: Array> = await k.raw( + 'SELECT COLUMN_NAME AS name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION LIMIT 65', + [table] + ) + return rows.map(row => row.name) +} + +interface SnapshotContext { + storage: StorageKnex + userId: number + snapshotId: string + profileIndexes: SnapshotIndexState + relationIndexes: SnapshotIndexState + certificateIndexes: SnapshotIndexState + globalIndexes: SnapshotIndexState + columns: Map +} + +/** Select a complete prefix without fetching payloads or skipping oversized rows. */ +function boundedPrefix( + candidates: Array>, + maxBytes: number +): { count: number; payloadBytes: number } { + let payloadBytes = 0 + let count = 0 + for (const candidate of candidates) { + if (Number(candidate.__snapshotOwned) !== 1) { + throw new WERR_INVALID_OPERATION('Snapshot relation does not belong to this wallet profile') + } + const bytes = Number(candidate.__snapshotBytes) + if (!Number.isSafeInteger(bytes) || bytes < 0) throw new WERR_INVALID_OPERATION('Invalid snapshot row size') + if (payloadBytes + bytes > maxBytes) { + if (count === 0) + throw new SnapshotResourceLimitError('Snapshot row exceeds maxBytes; large-value streaming is required') + break + } + payloadBytes += bytes + count++ + } + return { count, payloadBytes } +} + +async function readPage( + context: SnapshotContext, + trx: TrxToken, + table: T, + after: Array | undefined, + limits: { maxRows: number; maxBytes: number } +): Promise> { + const { storage, userId, columns, snapshotId, profileIndexes, relationIndexes, certificateIndexes, globalIndexes } = + context + const k = storage.toDb(trx) + const schema = definitions[table] + let fields = columns.get(table) + if (fields === undefined) { + fields = await columnNames(k, schema.name, storage.dbtype === 'MySQL') + if (fields.length === 0 || fields.length > 64) throw new WERR_INVALID_OPERATION('Unsupported snapshot schema') + columns.set(table, fields) + } + let orderKeys = schema.keys + if (profileIndexes && auxiliaryTableIds[table] !== undefined) orderKeys = ['snapshotRowId'] + if (relationIndexes && auxiliaryRelationTableIds[table] !== undefined) + orderKeys = ['snapshotLeftId', 'snapshotRightId'] + if (certificateIndexes && table === 'certificateFields') orderKeys = ['snapshotFieldName', 'snapshotCertificateId'] + if (globalIndexes && (table === 'provenTxs' || table === 'provenTxReqs')) orderKeys = ['rowId'] + const base = (): Knex.QueryBuilder => { + const q = walletSnapshotSourceQuery( + k, + table, + userId, + profileIndexes, + relationIndexes, + certificateIndexes, + globalIndexes + ) + if (after !== undefined) seek(q, orderKeys, after, false, storage.dbtype === 'MySQL') + for (const key of orderKeys) void q.orderBy(key) + return q + } + const keyColumns = schema.keys.map(key => keyColumn(k, key, storage.dbtype === 'MySQL')) + // Only bounded keys and lengths cross the driver boundary in this pass. A + // giant blob/history must not be loaded merely to discover it exceeds budget. + const candidates: Array> = await base() + .select(...keyColumns, charge(k, fields, storage.dbtype === 'MySQL'), relationGuard(k, table, userId)) + .limit(limits.maxRows) + const { count, payloadBytes } = boundedPrefix(candidates, limits.maxBytes) + const last = candidates[count - 1] + const cursor: WalletSnapshotCursor | undefined = + last === undefined ? undefined : { version: 1, snapshotId, table, after: schema.keys.map(key => last[key]) } + // Validate database keys too, including SQLite strings without enforced SQL length. + const end = position(cursor, snapshotId, table, schema.keys) + let rows: Array> = [] + if (end !== undefined) { + const query = base().select(`${schema.name}.*`).limit(count) + seek(query, orderKeys, end, true, storage.dbtype === 'MySQL') + const raw: Array> = await query + rows = raw.map(row => normalize(storage, row, schema)) + } + return { + rows, + payloadBytes, + cursor, + done: count === candidates.length && candidates.length < limits.maxRows + } +} + +// Keep the existing optional positional call contract while grouping the four +// index-generation selections captured by the same retained view. +type SnapshotIndexFlags = [ + profileIndexes?: SnapshotIndexState, + relationIndexes?: SnapshotIndexState, + certificateIndexes?: SnapshotIndexState, + globalIndexes?: SnapshotIndexState +] + +/** Bind every page to one provider-owned view and immutable profile identifiers. */ +export function createKnexWalletSnapshotPageReader( + storage: StorageKnex, + userId: number, + snapshotId: string, + view: RetainedReadSnapshot, + ...[ + profileIndexes = false, + relationIndexes = false, + certificateIndexes = false, + globalIndexes = false + ]: SnapshotIndexFlags +): WalletReadSnapshot['readPage'] { + const context: SnapshotContext = { + storage, + userId, + snapshotId, + profileIndexes, + relationIndexes, + certificateIndexes, + globalIndexes, + columns: new Map() + } + return async ( + table: T, + cursor?: WalletSnapshotCursor, + limits: WalletSnapshotPageLimits = {} + ): Promise> => { + const schema = definition(table) + const after = position(cursor, snapshotId, table, schema.keys) + const maxRows = bound(limits.maxRows, 128, 1000, 'maxRows') + const maxBytes = bound(limits.maxBytes, 262144, 16777216, 'maxBytes') + return await view.read(trx => readPage(context, trx, table, after, { maxRows, maxBytes })) + } +} + +/** SQL implementation, deliberately separate from legacy OFFSET sync and public RPC. */ +export async function openKnexWalletReadSnapshot( + storage: StorageKnex, + identityKey: string, + options: WalletReadSnapshotOptions +): Promise { + if (typeof identityKey !== 'string' || !/^(02|03)[0-9a-fA-F]{64}$/.test(identityKey)) { + throw new WERR_INVALID_PARAMETER('identityKey', 'a compressed public identity key') + } + const view = await storage.openReadSnapshot(options) + try { + const { header, profileIndexes, relationIndexes, certificateIndexes, globalIndexes } = await view.read( + async trx => { + const generation = await readGenerationIndexState(storage.toDb(trx), storage.knex.client.config.migrations) + const sourceStorage = await storage.readSettings(trx) + const user = await storage.findUserByIdentityKey(identityKey, trx) + if (user === undefined) throw new WERR_INVALID_PARAMETER('identityKey', 'an existing wallet profile') + return { + header: { sourceStorage, user }, + profileIndexes: + generation ?? + (await readSnapshotProfileIndexState(storage.toDb(trx), storage.knex.client.config.migrations)), + relationIndexes: + generation ?? + (await readSnapshotRelationIndexState(storage.toDb(trx), storage.knex.client.config.migrations)), + globalIndexes: + generation ?? + (await readSnapshotGlobalIndexState(storage.toDb(trx), storage.knex.client.config.migrations)), + certificateIndexes: + generation ?? + (await readSnapshotCertificateIndexState(storage.toDb(trx), storage.knex.client.config.migrations)) + } + } + ) + const userId = header.user.userId + const snapshotId = Utils.toHex(Random(32)) + return { + version: 1, + snapshotId, + ...header, + expiresAt: view.expiresAt, + get isOpen() { + return view.isOpen + }, + closed: view.closed, + close: view.close, + readPage: createKnexWalletSnapshotPageReader( + storage, + userId, + snapshotId, + view, + profileIndexes, + relationIndexes, + certificateIndexes, + globalIndexes + ) + } + } catch (error) { + // Keep the opening error authoritative while still awaiting physical cleanup. + await view.close().catch(() => undefined) + throw error + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/PostgresSnapshotCompatibility.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/PostgresSnapshotCompatibility.test.ts new file mode 100644 index 000000000..3f2fa5690 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/PostgresSnapshotCompatibility.test.ts @@ -0,0 +1,58 @@ +import { knex, type Knex } from 'knex' +import { KnexMigrations } from '../schema/KnexMigrations' +import { forSnapshotSqlDialect } from '../schema/snapshotSqlMigration' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' + +const entries = [ + '2026-09-30-001 add durable snapshot sync', + '2026-09-30-002 add snapshot archive staging', + '2026-09-30-003 add snapshot archive requests', + '2026-10-01-001 add snapshot archive source owners', + '2026-10-01-002 add snapshot archive source guards' +] + +test('Postgres snapshot auxiliary migrations never execute SQLite or MySQL SQL', async () => { + const k = knex({ client: 'pg', connection: { database: 'unused' }, pool: { min: 0, max: 1 } }) + const source = new KnexMigrations('test', 'postgres compatibility', '1'.repeat(64), 10000) + const names = (await source.getMigrations()).filter(name => name.includes('snapshot')) + expect(names).toHaveLength(10) + expect(names).toEqual(expect.arrayContaining(entries)) + const query = jest.spyOn(k.client, 'query').mockRejectedValue(new Error('Snapshot SQL reached Postgres')) + const acquire = jest.spyOn(k.client, 'acquireConnection').mockRejectedValue(new Error('Snapshot opened Postgres')) + try { + for (const name of names) { + const migration = await source.getMigration(name) + await migration.up(k) + await migration.down?.(k) + } + expect(query).not.toHaveBeenCalled() + expect(acquire).not.toHaveBeenCalled() + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + expect(storage.supportsReadSnapshot()).toBe(false) + expect(storage.supportsRetainedReadSnapshot()).toBe(false) + expect(storage.supportsWalletReadSnapshot()).toBe(false) + expect(storage.getSnapshotSync()).toBeUndefined() + expect(await storage.supportsSnapshotArchiveSource()).toBe(false) + await storage.destroy() + } finally { + query.mockRestore() + acquire.mockRestore() + await k.destroy() + } +}) + +test.each(['better-sqlite3', 'mysql2'])( + '%s keeps its existing snapshot migration lifecycle and failures', + async client => { + const k = knex({ client, connection: {}, pool: { min: 0, max: 1 } }) + try { + const error = new Error('Existing migration failure') + const migrate = jest.fn, [Knex]>().mockRejectedValue(error) + await expect(forSnapshotSqlDialect(migrate)(k)).rejects.toBe(error) + expect(migrate).toHaveBeenCalledWith(k) + } finally { + await k.destroy() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.property.test.ts new file mode 100644 index 000000000..a6a132db4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.property.test.ts @@ -0,0 +1,440 @@ +import { + addSnapshotProfileIndexes, + removeSnapshotProfileIndexes, + snapshotProfileTables +} from '../schema/snapshotProfileIndexMigration' +import fc from 'fast-check' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import type { WalletSnapshotCursor } from './WalletReadSnapshot' +import { runInSeries } from '../../utility/runInSeries' +import { retainReadSnapshot } from './RetainedReadSnapshot' +import { addSnapshotRelationIndexes } from '../schema/snapshotRelationIndexMigration' +import { + expectRelationMembership, + minimalRelationDatabase, + relationFixtures +} from '../../../test/utils/snapshotRelationFixtures' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +test('generated relation writes preserve OR ownership through parent moves, rekeys, tombstones, rollback and restart', async () => { + const k = await minimalRelationDatabase() + try { + await addSnapshotRelationIndexes(k) + await fc.assert( + fc.asyncProperty( + fc.array( + fc.record({ + pair: fc.integer({ min: 0, max: 1 }), + left: fc.integer({ min: 1, max: 5 }), + right: fc.integer({ min: 1, max: 5 }), + owner: fc.integer({ min: 1, max: 3 }), + kind: fc.constantFrom( + 'left', + 'right', + 'map', + 'tombstone', + 'delete-map', + 'delete-left', + 'delete-right', + 'left-key', + 'right-key', + 'map-key', + 'rollback', + 'repeat' + ) + }), + { minLength: 1, maxLength: 25 } + ), + async schedule => { + await runInSeries(relationFixtures, async p => { + await k(p.table).delete() + await k(p.left).delete() + await k(p.right).delete() + }) + await runInSeries(schedule, async op => { + const p = relationFixtures[op.pair] + if (op.kind === 'left') + await k(p.left) + .insert({ [p.leftKey]: op.left, userId: op.owner }) + .onConflict(p.leftKey) + .merge() + else if (op.kind === 'right') + await k(p.right) + .insert({ [p.rightKey]: op.right, userId: op.owner }) + .onConflict(p.rightKey) + .merge() + else if (op.kind === 'map') + await k(p.table) + .insert({ [p.leftKey]: op.left, [p.rightKey]: op.right, isDeleted: false }) + .onConflict([p.leftKey, p.rightKey]) + .ignore() + else if (op.kind === 'tombstone') + await k(p.table) + .where({ [p.leftKey]: op.left, [p.rightKey]: op.right }) + .update({ isDeleted: true }) + else if (op.kind === 'delete-map') + await k(p.table) + .where({ [p.leftKey]: op.left, [p.rightKey]: op.right }) + .delete() + else if (op.kind === 'delete-left') await k(p.left).where(p.leftKey, op.left).delete() + else if (op.kind === 'delete-right') await k(p.right).where(p.rightKey, op.right).delete() + else if (op.kind === 'left-key' || op.kind === 'right-key' || op.kind === 'map-key') { + try { + if (op.kind === 'left-key') + await k(p.left) + .where(p.leftKey, op.left) + .update({ [p.leftKey]: (op.left % 5) + 1 }) + else if (op.kind === 'right-key') + await k(p.right) + .where(p.rightKey, op.right) + .update({ [p.rightKey]: (op.right % 5) + 1 }) + else + await k(p.table) + .where({ [p.leftKey]: op.left, [p.rightKey]: op.right }) + .update({ [p.leftKey]: (op.left % 5) + 1, [p.rightKey]: (op.right % 5) + 1 }) + } catch (error) { + if ((error as { code?: string }).code !== 'SQLITE_CONSTRAINT_PRIMARYKEY') throw error + } + } else if (op.kind === 'repeat') await addSnapshotRelationIndexes(k) + else { + const failure = new Error('synthetic relation rollback') + await expect( + k.transaction(async trx => { + await trx(p.left) + .insert({ [p.leftKey]: op.left, userId: op.owner }) + .onConflict(p.leftKey) + .merge() + throw failure + }) + ).rejects.toBe(failure) + } + await expectRelationMembership(k) + }) + } + ) + ) + } finally { + await k.destroy() + } +}) + +function gate() { + let resolve!: () => void + const promise = new Promise(yes => { + resolve = yes + }) + return { promise, resolve } +} + +type Outcome = { ok: true; value: number } | { ok: false; error: unknown } +interface PendingRead { + value: number + release: () => void + outcome: Promise + settled: () => boolean +} + +const operations = fc.array(fc.constantFrom('read', 'settle', 'close', 'cancel', 'advance'), { + minLength: 1, + maxLength: 40 +}) + +afterEach(() => { + jest.useRealTimers() +}) + +test('random profile rows and page budgets preserve the pinned keyset under independent writes', async () => { + const directory = await mkdtemp(join(tmpdir(), 'wallet-page-property-')) + const open = () => + new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) + const source = open() + const writer = open() + const identity = '02' + '11'.repeat(32) + try { + await source.knex.raw('PRAGMA journal_mode = WAL') + await source.migrate('property source', 'property-storage') + await source.makeAvailable() + const { user } = await source.findOrInsertUser(identity) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await writer.makeAvailable() + await fc.assert( + fc.asyncProperty( + fc.array(fc.record({ foreign: fc.boolean(), deleted: fc.boolean(), length: fc.integer({ min: 0, max: 80 }) }), { + maxLength: 40 + }), + fc.integer({ min: 1, max: 12 }), + fc.integer({ min: 1100, max: 8192 }), + async (entries, maxRows, maxBytes) => { + await source.knex('tx_labels').del() + const when = new Date('2026-01-01T00:00:00.000Z') + const records = entries.map((entry, index) => ({ + txLabelId: index + 1, + userId: entry.foreign ? other.userId : user.userId, + label: `${index}-${'é'.repeat(entry.length)}`, + isDeleted: entry.deleted, + created_at: when, + updated_at: when + })) + if (records.length > 0) + await source + .knex('tx_labels') + .insert(records.map(row => ({ ...row, created_at: when.toISOString(), updated_at: when.toISOString() }))) + const expected = records.filter(row => row.userId === user.userId) + const view = await source.openWalletReadSnapshot(identity) + try { + await writer.knex('tx_labels').where({ userId: user.userId }).update({ isDeleted: true }) + await writer.findOrInsertTxLabel(user.userId, 'post-snapshot insert') + const actual: typeof expected = [] + let cursor: WalletSnapshotCursor | undefined + for (let pages = 0; ; pages++) { + expect(pages).toBeLessThanOrEqual(expected.length + 1) + const page = await view.readPage('txLabels', cursor, { maxRows, maxBytes }) + expect(page.rows.length).toBeLessThanOrEqual(maxRows) + expect(page.payloadBytes).toBeLessThanOrEqual(maxBytes) + expect(await view.readPage('txLabels', cursor, { maxRows, maxBytes })).toEqual(page) + actual.push(...page.rows) + if (page.done) break + expect(page.rows.length).toBeGreaterThan(0) + cursor = page.cursor + } + expect(actual).toEqual(expected) + expect(new Set(actual.map(row => row.txLabelId)).size).toBe(expected.length) + } finally { + await view.close() + } + } + ) + ) + } finally { + await Promise.all([source.destroy(), writer.destroy()]) + await rm(directory, { recursive: true, force: true }) + } +}) + +test('random schedules preserve single-read admission, late-result rejection and physical cleanup ownership', async () => { + jest.useFakeTimers() + await fc.assert( + fc.asyncProperty(operations, fc.integer({ min: 1, max: 1000 }), async (steps, lifetimeMs) => { + const controller = new AbortController() + const cleanup = gate() + const token = { synthetic: true } + let physicalReads = 0 + let enteredReads = 0 + let leftCallback = false + let released = false + let closed = false + let pending: PendingRead | undefined + let expectedReads = 0 + let alive = true + let elapsed = 0 + const lifetime = retainReadSnapshot( + async read => { + try { + await read(token) + } finally { + leftCallback = true + await cleanup.promise + released = true + } + }, + async received => { + expect(received).toBe(token) + }, + { signal: controller.signal, lifetimeMs } + ) + void lifetime.closed.catch(() => undefined) + const view = await lifetime.opened.catch(async error => { + cleanup.resolve() + await lifetime.closed.catch(() => undefined) + throw error + }) + void view.closed.then( + () => { + closed = true + }, + () => undefined + ) + + const finishRead = async (): Promise => { + if (pending === undefined) return + const active = pending + active.release() + const outcome = await active.outcome + if (alive) expect(outcome).toEqual({ ok: true, value: active.value }) + else expect(outcome).toEqual({ ok: false, error: expect.any(Error) }) + pending = undefined + expect(physicalReads).toBe(0) + } + try { + // Each generated operation awaits the previous operation's observations; + // the database read itself remains independently held until `settle`. + await runInSeries(steps, async step => { + if (step === 'read') { + if (!alive || pending !== undefined) { + const extra = jest.fn(async () => 1) + await expect(view.read(extra)).rejects.toBeInstanceOf(Error) + expect(extra).not.toHaveBeenCalled() + } else { + const held = gate() + const value = ++expectedReads + let settled = false + const outcome: Promise = view + .read(async received => { + expect(received).toBe(token) + physicalReads++ + enteredReads++ + expect(physicalReads).toBe(1) + await held.promise + physicalReads-- + return value + }) + .then( + result => { + settled = true + return { ok: true, value: result } + }, + error => { + settled = true + return { ok: false, error } + } + ) + pending = { value, release: held.resolve, outcome, settled: () => settled } + } + } else if (step === 'settle') { + await finishRead() + } else if (step === 'advance') { + elapsed += 250 + if (elapsed >= lifetimeMs) alive = false + jest.advanceTimersByTime(250) + } else { + alive = false + if (step === 'cancel') controller.abort() + else void view.close().catch(() => undefined) + } + // Observe both read continuation and provider callback handoffs. + await Promise.resolve() + await Promise.resolve() + expect(view.isOpen).toBe(alive) + expect(enteredReads).toBe(expectedReads) + expect(released).toBe(false) + expect(closed).toBe(false) + if (pending !== undefined) { + expect(pending.settled()).toBe(false) + expect(physicalReads).toBe(1) + expect(leftCallback).toBe(false) + } + }) + await finishRead() + } finally { + const closing = view.close() + pending?.release() + await pending?.outcome + cleanup.resolve() + await closing + } + expect(physicalReads).toBe(0) + expect(released).toBe(true) + expect(closed).toBe(true) + expect(view.isOpen).toBe(false) + expect(jest.getTimerCount()).toBe(0) + }) + ) +}) + +test('generated source writes, profile moves, rollback and restart preserve exact auxiliary membership', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + try { + await runInSeries(snapshotProfileTables, async ({ table, key }) => { + await k.schema.createTable(table, columns => { + columns.integer(key).primary() + columns.integer('userId').notNullable() + columns.text('value') + }) + }) + await addSnapshotProfileIndexes(k) + const verify = async (): Promise => { + const expected: Array<{ snapshotTableId: number; snapshotUserId: number; snapshotRowId: number }> = [] + await runInSeries(snapshotProfileTables.entries(), async ([snapshotTableId, { table, key }]) => { + const rows: Array> = await k(table).select(key, 'userId') + expected.push(...rows.map(row => ({ snapshotTableId, snapshotUserId: row.userId, snapshotRowId: row[key] }))) + }) + expected.sort( + (a, b) => + a.snapshotTableId - b.snapshotTableId || + a.snapshotUserId - b.snapshotUserId || + a.snapshotRowId - b.snapshotRowId + ) + expect(await k('snapshot_profile_keys').orderBy(['snapshotTableId', 'snapshotUserId', 'snapshotRowId'])).toEqual( + expected + ) + } + await fc.assert( + fc.asyncProperty( + fc.array( + fc.record({ + tableId: fc.integer({ min: 0, max: 7 }), + rowId: fc.integer({ min: 1, max: 20 }), + owner: fc.integer({ min: 1, max: 4 }), + kind: fc.constantFrom('insert-or-update', 'delete', 'rollback', 'rebuild', 'repeat'), + value: fc.string({ maxLength: 12 }) + }), + { minLength: 1, maxLength: 12 } + ), + async schedule => { + await runInSeries(snapshotProfileTables, async ({ table }) => { + await k(table).delete() + }) + await verify() + await runInSeries(schedule, async operation => { + const { table, key } = snapshotProfileTables[operation.tableId] + const row = { [key]: operation.rowId, userId: operation.owner, value: operation.value } + if (operation.kind === 'insert-or-update') await k(table).insert(row).onConflict(key).merge() + else if (operation.kind === 'delete') await k(table).where(key, operation.rowId).delete() + else if (operation.kind === 'rollback') { + const rollback = new Error('synthetic rollback') + await expect( + k.transaction(async trx => { + await trx(table).insert(row).onConflict(key).merge() + throw rollback + }) + ).rejects.toBe(rollback) + } else if (operation.kind === 'rebuild') { + await removeSnapshotProfileIndexes(k) + await addSnapshotProfileIndexes(k) + } else await addSnapshotProfileIndexes(k) + await verify() + }) + } + ) + ) + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.test.ts new file mode 100644 index 000000000..1b2b0a8ce --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.test.ts @@ -0,0 +1,353 @@ +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' +import { retainReadSnapshot, type RetainedReadSnapshotLifetime } from './RetainedReadSnapshot' + +function gate() { + let resolve!: () => void + const promise = new Promise(yes => { + resolve = yes + }) + return { promise, resolve } +} + +const lifetimes: RetainedReadSnapshotLifetime[] = [] +const gates: Array> = [] +const expectedFailures = new Set() + +function hold() { + const value = gate() + gates.push(value) + return value +} + +function fixture( + options: Parameters[2] = {}, + controls: { + acquire?: Promise + initialize?: Promise + cleanup?: Promise + initializeError?: Error + beginError?: Error + cleanupError?: Error + } = {} +) { + const token: TrxToken = { synthetic: true } + const events: string[] = [] + const initializing = gate() + const run = jest.fn(async (read: (trx: TrxToken) => Promise) => { + await controls.acquire + events.push('acquired') + try { + if (controls.beginError) throw controls.beginError + await read(token) + events.push('commit') + } finally { + await controls.cleanup + events.push('released') + } + if (controls.cleanupError) throw controls.cleanupError + }) + const initialize = jest.fn(async (trx: TrxToken) => { + expect(trx).toBe(token) + events.push('initialize') + initializing.resolve() + await controls.initialize + if (controls.initializeError) throw controls.initializeError + }) + const lifetime = retainReadSnapshot(run, initialize, options) + // Preserve the promise for assertions while ensuring failed-test cleanup + // cannot create an unrelated unhandled rejection from a still-opening view. + void lifetime.opened.catch(() => undefined) + void lifetime.closed.catch(() => undefined) + lifetimes.push(lifetime) + return { token, events, run, initialize, initializing: initializing.promise, lifetime } +} + +afterEach(async () => { + gates.splice(0).forEach(value => value.resolve()) + const results = await Promise.allSettled(lifetimes.splice(0).map(value => value.close())) + for (const result of results) { + if (result.status === 'rejected') expect(expectedFailures.has(result.reason)).toBe(true) + } + expectedFailures.clear() + jest.useRealTimers() + jest.restoreAllMocks() +}) + +test('establishes the read view before opening and retains the exact token across idle boundaries', async () => { + const initialize = hold() + const f = fixture({}, { initialize: initialize.promise }) + let opened = false + void f.lifetime.opened.then( + () => { + opened = true + }, + () => undefined + ) + await Promise.resolve() + expect(opened).toBe(false) + initialize.resolve() + const view = await f.lifetime.opened + expect(f.events).toEqual(['acquired', 'initialize']) + expect(view.isOpen).toBe(true) + expect(await view.read(async token => token)).toBe(f.token) + await new Promise(resolve => setImmediate(resolve)) + expect(await view.read(async token => token)).toBe(f.token) + await Promise.all([view.close(), view.close(), view.closed]) + expect(view.isOpen).toBe(false) + expect(f.events).toEqual(['acquired', 'initialize', 'commit', 'released']) + await expect(view.read(async () => 1)).rejects.toThrow('closed') +}) + +test('refuses concurrent and nested reads without queuing or invoking them', async () => { + const f = fixture() + const view = await f.lifetime.opened + const releaseRead = hold() + const first = view.read(async () => await releaseRead.promise) + void first.catch(() => undefined) + const extra = jest.fn(async () => 2) + await expect(view.read(extra)).rejects.toThrow('read in flight') + expect(extra).not.toHaveBeenCalled() + releaseRead.resolve() + await first + await view.read(async () => { + await expect(view.read(extra)).rejects.toThrow('read in flight') + }) + expect(extra).not.toHaveBeenCalled() +}) + +test.each(['close', 'abort', 'expiry'] as const)( + '%s discards a late read and retains capacity until physical read and cleanup both settle', + async action => { + jest.useFakeTimers() + const controller = new AbortController() + const releaseRead = hold() + const cleanup = hold() + const f = fixture({ lifetimeMs: 100, signal: controller.signal }, { cleanup: cleanup.promise }) + const view = await f.lifetime.opened + let released = false + void view.closed.then( + () => { + released = true + }, + () => undefined + ) + const pending = view.read(async () => { + await releaseRead.promise + return 'late bytes' + }) + const outcome = pending.catch(error => error) + if (action === 'close') void view.close().catch(() => undefined) + if (action === 'abort') controller.abort() + if (action === 'expiry') await jest.advanceTimersByTimeAsync(100) + expect(view.isOpen).toBe(false) + expect(released).toBe(false) + expect(f.events).toEqual(['acquired', 'initialize']) + const another = jest.fn(async () => 2) + await expect(view.read(another)).rejects.toThrow(/closed|cancelled|expired/) + expect(another).not.toHaveBeenCalled() + releaseRead.resolve() + expect(await outcome).toBeInstanceOf(Error) + await Promise.resolve() + expect(released).toBe(false) + cleanup.resolve() + await view.closed + expect(f.events).toEqual(['acquired', 'initialize', 'commit', 'released']) + expect(released).toBe(true) + expect(jest.getTimerCount()).toBe(0) + } +) + +test.each(['acquire', 'initialize'] as const)( + 'cancelling during %s does not release an unsettled transaction', + async phase => { + const controller = new AbortController() + const held = hold() + const f = fixture({ signal: controller.signal }, { [phase]: held.promise }) + const opening = f.lifetime.opened.catch(error => error) + let closed = false + void f.lifetime.closed.then( + () => { + closed = true + }, + () => undefined + ) + if (phase === 'initialize') await f.initializing + else await Promise.resolve() + controller.abort() + expect(await opening).toBeInstanceOf(Error) + expect(closed).toBe(false) + held.resolve() + await f.lifetime.closed + expect(closed).toBe(true) + if (phase === 'acquire') expect(f.initialize).not.toHaveBeenCalled() + expect(f.events.at(-1)).toBe('released') + } +) + +test('pre-cancelled views never acquire a connection', async () => { + const controller = new AbortController() + controller.abort() + const f = fixture({ signal: controller.signal }) + await expect(f.lifetime.opened).rejects.toThrow('cancelled') + await f.lifetime.closed + expect(f.run).not.toHaveBeenCalled() +}) + +test.each([0, -1, 0.5, NaN, Infinity, 3_600_001])('refuses invalid lifetime %s before acquisition', lifetimeMs => { + const run = jest.fn() + expect(() => { + const lifetime = retainReadSnapshot(run, async () => undefined, { lifetimeMs }) + void lifetime.opened.catch(() => undefined) + void lifetime.closed.catch(() => undefined) + lifetimes.push(lifetime) + }).toThrow('an integer from 1 to 3600000') + expect(run).not.toHaveBeenCalled() +}) + +test('monotonic expiry rejects a read even before its timer callback executes', async () => { + const f = fixture({ lifetimeMs: 100 }) + const view = await f.lifetime.opened + jest.spyOn(performance, 'now').mockReturnValue(Number.MAX_SAFE_INTEGER) + const read = jest.fn(async () => 1) + await expect(view.read(read)).rejects.toThrow('expired') + expect(read).not.toHaveBeenCalled() + await view.closed +}) + +test.each(['begin', 'initialize', 'read', 'cleanup'] as const)( + 'preserves %s errors and always finishes physical cleanup', + async phase => { + const error = new Error(`synthetic ${phase} failure`) + expectedFailures.add(error) + const f = fixture( + {}, + { + initializeError: phase === 'initialize' ? error : undefined, + beginError: phase === 'begin' ? error : undefined, + cleanupError: phase === 'cleanup' ? error : undefined + } + ) + if (phase === 'begin' || phase === 'initialize') { + await expect(f.lifetime.opened).rejects.toBe(error) + } else { + const view = await f.lifetime.opened + if (phase === 'read') { + await expect( + view.read(() => { + throw error + }) + ).rejects.toBe(error) + } else { + await expect(view.close()).rejects.toBe(error) + } + } + await expect(f.lifetime.closed).rejects.toBe(error) + expect(f.events.at(-1)).toBe('released') + } +) + +test('rejects an invalid callback without poisoning the retained view', async () => { + const f = fixture() + const view = await f.lifetime.opened + await expect(view.read(undefined as never)).rejects.toThrow('callback') + expect(await view.read(async () => 19)).toBe(19) +}) + +test('wall-clock expiry also covers suspension before the monotonic clock or timer advances', async () => { + const f = fixture({ lifetimeMs: 100 }) + const view = await f.lifetime.opened + jest.spyOn(Date, 'now').mockReturnValue(view.expiresAt) + expect(view.isOpen).toBe(false) + await expect(view.read(async () => 1)).rejects.toThrow('expired') + await view.closed +}) + +test.each(['acquire', 'initialize'] as const)( + 'expiry during %s retains capacity through delayed physical cleanup', + async phase => { + jest.useFakeTimers() + const held = hold() + const f = fixture({ lifetimeMs: 100 }, { [phase]: held.promise }) + const opening = f.lifetime.opened.catch(error => error) + let closed = false + void f.lifetime.closed.then( + () => { + closed = true + }, + () => undefined + ) + if (phase === 'initialize') await f.initializing + else await Promise.resolve() + jest.advanceTimersByTime(100) + expect(await opening).toEqual(expect.objectContaining({ message: expect.stringContaining('expired') })) + expect(closed).toBe(false) + held.resolve() + await f.lifetime.closed + expect(closed).toBe(true) + expect(f.events.at(-1)).toBe('released') + } +) + +// Inclusive limits and default lifetime are part of the public admission contract. +test.each([1, 3_600_000, undefined])( + 'accepts the lifetime boundary %s and reports its exact expiry', + async lifetimeMs => { + jest.useFakeTimers() + const start = Date.now() + const f = fixture({ lifetimeMs }) + const view = await f.lifetime.opened + expect(view.expiresAt).toBe(start + (lifetimeMs ?? 300_000)) + expect(view.isOpen).toBe(true) + await view.close() + expect(jest.getTimerCount()).toBe(0) + } +) + +test('monotonic expiry includes its exact boundary despite a backward wall-clock adjustment', async () => { + const start = performance.now() + jest.spyOn(performance, 'now').mockReturnValue(start) + const f = fixture({ lifetimeMs: 100 }) + const view = await f.lifetime.opened + jest.spyOn(Date, 'now').mockReturnValue(view.expiresAt - 500) + jest.spyOn(performance, 'now').mockReturnValue(start + 100) + await expect(view.read(async () => 1)).rejects.toThrow('expired') + await view.closed +}) + +test('keeps the first stop reason across repeated cancellation and close, and removes its signal listener', async () => { + const controller = new AbortController() + const remove = jest.spyOn(controller.signal, 'removeEventListener') + const f = fixture({ signal: controller.signal }) + const view = await f.lifetime.opened + controller.abort() + await view.close() + await expect(view.read(async () => 1)).rejects.toThrow('cancelled') + expect(remove).toHaveBeenCalledWith('abort', expect.any(Function)) + expect(remove).toHaveBeenCalledTimes(1) +}) + +test('acquisition failure clears its expiry timer and removes its signal listener', async () => { + jest.useFakeTimers() + const controller = new AbortController() + const remove = jest.spyOn(controller.signal, 'removeEventListener') + const failure = new Error('acquire failed') + expectedFailures.add(failure) + const f = fixture({ signal: controller.signal }, { beginError: failure }) + await expect(f.lifetime.opened).rejects.toBe(failure) + await expect(f.lifetime.closed).rejects.toBe(failure) + expect(jest.getTimerCount()).toBe(0) + expect(remove).toHaveBeenCalledWith('abort', expect.any(Function)) +}) + +test('closing before consuming the opening promise remains observed and preserves the opening rejection', async () => { + const lifetime = retainReadSnapshot( + async read => await read({ synthetic: true }), + async () => undefined + ) + lifetimes.push(lifetime) + // The owner may drain a cancelled opening before its opening consumer resumes. + // Unlike fixture(), deliberately attach no observer to opened until a later turn. + await lifetime.close() + await new Promise(resolve => setImmediate(resolve)) + await expect(lifetime.opened).rejects.toThrow('closed') +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.ts new file mode 100644 index 000000000..f07f5ab0c --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/RetainedReadSnapshot.ts @@ -0,0 +1,172 @@ +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import { SnapshotCancelledError } from './SnapshotCancelledError' +import type { TrxToken } from '../../sdk/WalletStorage.interfaces' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' + +/** Local database view only. This is not a remote export or authorization token. */ +export interface RetainedReadSnapshotOptions { + /** Total lifetime, including connection acquisition. Defaults to five minutes; at most one hour. */ + lifetimeMs?: number + /** Stops admission and discards late results; physical database work must drain before release. */ + signal?: AbortSignal +} + +export interface RetainedReadSnapshot { + /** Informational wall-clock expiry. Admission also enforces a monotonic lifetime. */ + readonly expiresAt: number + readonly isOpen: boolean + /** Settles only after the provider has finished its transaction and connection cleanup. */ + readonly closed: Promise + /** + * One database read at a time. Await every query before returning; the token + * is valid only inside the callback. Do not + * write, perform peer/file I/O, or await close() from inside this callback. + * Nested/concurrent reads reject rather than creating an unbounded queue. + */ + read: (read: (trx: TrxToken) => Promise) => Promise + /** Stop admission, discard late results, and await physical transaction cleanup. Idempotent. */ + close: () => Promise +} + +interface Deferred { + promise: Promise + resolve: (value: T) => void + reject: (reason: unknown) => void +} + +function deferred(): Deferred { + let resolve!: (value: T) => void + let reject!: (reason: unknown) => void + const promise = new Promise((yes, no) => { + resolve = yes + reject = no + }) + return { promise, resolve, reject } +} + +/** Provider-owned lifecycle; retain the admission slot until `closed` settles, even if opening is cancelled. */ +export interface RetainedReadSnapshotLifetime { + opened: Promise + closed: Promise + close: () => Promise +} + +export function retainReadSnapshot( + run: (read: (trx: TrxToken) => Promise, assertActive: () => void) => Promise, + establishView: (trx: TrxToken) => Promise, + options: RetainedReadSnapshotOptions = {} +): RetainedReadSnapshotLifetime { + const lifetimeMs = options.lifetimeMs ?? 300_000 + if (!Number.isSafeInteger(lifetimeMs) || lifetimeMs < 1 || lifetimeMs > 3_600_000) { + throw new WERR_INVALID_PARAMETER('lifetimeMs', 'an integer from 1 to 3600000') + } + const { signal } = options + const expiresAt = Date.now() + lifetimeMs + const startedAt = performance.now() + const opened = deferred() + const stopped = deferred() + const closed = deferred() + let token: TrxToken | undefined + let stopReason: WERR_INVALID_OPERATION | undefined + let readFailure: { error: unknown } | undefined + let inFlight: Promise | undefined + let busy = false + let timer: ReturnType | undefined + + // A provider can close a still-opening view before its opening consumer + // resumes. Observe automatic rejection of both lifecycle promises while + // preserving their original errors for consumers awaiting opened/closed. + void opened.promise.catch(() => undefined) + void closed.promise.catch(() => undefined) + + const stop = (reason: WERR_INVALID_OPERATION): void => { + if (stopReason !== undefined) return + stopReason = reason + if (timer !== undefined) clearTimeout(timer) + signal?.removeEventListener('abort', abort) + opened.reject(reason) + stopped.resolve() + } + const abort = (): void => stop(new SnapshotCancelledError('Retained read snapshot was cancelled')) + const checkDeadline = (): void => { + if (Date.now() >= expiresAt || performance.now() - startedAt >= lifetimeMs) { + stop(new SnapshotResourceLimitError('Retained read snapshot expired')) + } + } + const assertOpen = (): void => { + checkDeadline() + if (stopReason !== undefined) throw stopReason + } + const close = async (): Promise => { + stop(new WERR_INVALID_OPERATION('Retained read snapshot is closed')) + await closed.promise + } + const snapshot: RetainedReadSnapshot = { + expiresAt, + get isOpen() { + checkDeadline() + // The handle is delivered only after the provider has established its view. + return stopReason === undefined + }, + closed: closed.promise, + close, + async read(read: (trx: TrxToken) => Promise): Promise { + assertOpen() + if (busy) throw new WERR_INVALID_OPERATION('Retained read snapshot already has a read in flight') + if (typeof read !== 'function') throw new WERR_INVALID_PARAMETER('read', 'a database read callback') + busy = true + // Invoke immediately after admission. A synchronous throw is a failed + // physical read too; neither failure nor cancellation frees the slot early. + const pending = (async () => await read(token as TrxToken))() + inFlight = pending.then( + () => undefined, + error => { + readFailure = { error } + stop(new WERR_INVALID_OPERATION('Retained read snapshot read failed')) + } + ) + try { + const result = await pending + assertOpen() + return result + } finally { + busy = false + inFlight = undefined + } + } + } + + const finish = async (): Promise => { + try { + if (stopReason === undefined) { + await run(async trx => { + if (stopReason !== undefined) return + token = trx + await establishView(trx) + checkDeadline() + if (stopReason !== undefined) return + opened.resolve(snapshot) + await stopped.promise + await inFlight + if (readFailure !== undefined) throw readFailure.error + }, assertOpen) + } + closed.resolve() + } catch (error) { + opened.reject(error) + closed.reject(error) + } finally { + stop(new WERR_INVALID_OPERATION('Retained read snapshot is closed')) + token = undefined + } + } + + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted === true) abort() + if (stopReason === undefined) { + timer = setTimeout(() => stop(new SnapshotResourceLimitError('Retained read snapshot expired')), lifetimeMs) + } + // Let the provider reserve its capacity slot before any acquisition hook can re-enter. + void Promise.resolve().then(finish) + return { opened: opened.promise, closed: closed.promise, close } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCancelledError.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCancelledError.ts new file mode 100644 index 000000000..46b0a10b9 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCancelledError.ts @@ -0,0 +1,4 @@ +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' + +/** A local explicit signal cancellation, distinct from validation or cleanup failure. */ +export class SnapshotCancelledError extends WERR_INVALID_OPERATION {} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.integration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.integration.test.ts new file mode 100644 index 000000000..2bdd745f1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.integration.test.ts @@ -0,0 +1,240 @@ +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { runInSeries } from '../../utility/runInSeries' +import { seedArchiveClosure } from '../../../test/utils/snapshotArchiveFixtures' +import { snapshotArchiveTables } from './archive/SnapshotArchive' +import { openKnexSnapshotArchiveSource, type SnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import { + addSnapshotCertificateIndexes, + removeSnapshotCertificateIndexes, + SNAPSHOT_CERTIFICATE_INDEX_MIGRATION +} from '../schema/snapshotCertificateIndexMigration' + +const identity = '02' + '11'.repeat(32) +const stores: StorageKnex[] = [] +const writers: Knex[] = [] +const directories: string[] = [] +const dates = { created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-01T00:00:00.000Z' } +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-certificate-index-')) + directories.push(directory) + const options = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + } + const k = knex(options) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + stores.push(source) + await k.raw('PRAGMA journal_mode=WAL') + await migrateBeforeSqliteGeneration(source, 'certificate index fixture', 'synthetic-certificate-index') + await source.makeAvailable() + // Start with the immediately preceding schema on both old and new source. + await removeSnapshotCertificateIndexes(k) + await k('knex_migrations').where('name', SNAPSHOT_CERTIFICATE_INDEX_MIGRATION).delete() + const { user } = await source.findOrInsertUser(identity) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, other.userId) + const writer = knex(options) + writers.push(writer) + return { source, writer, k, userId: user.userId, otherId: other.userId } +} +async function journal(k: Knex): Promise { + await k('knex_migrations').insert({ + name: SNAPSHOT_CERTIFICATE_INDEX_MIGRATION, + batch: 99, + migration_time: new Date() + }) +} +async function pages(view: WalletReadSnapshot | SnapshotArchiveSource) { + expect(Object.hasOwn(view, 'certificateIndexes')).toBe(false) + if ('validateClosure' in view) await view.validateClosure() + const result: Record = {} + await runInSeries(snapshotArchiveTables, async table => { + let cursor: WalletSnapshotCursor | undefined + const selected: unknown[] = [] + let complete = false + for (let pageNumber = 0; pageNumber < 25 && !complete; pageNumber++) { + const page = await view.readPage(table, cursor, { maxRows: 1, maxBytes: 131072 }) + selected.push({ rows: page.rows, after: page.cursor?.after, payloadBytes: page.payloadBytes, done: page.done }) + complete = page.done + cursor = page.cursor + } + expect(complete).toBe(true) + result[table] = selected + }) + return result +} +afterEach(async () => { + await runInSeries(writers.splice(0), k => k.destroy()) + await runInSeries(stores.splice(0), source => source.destroy()) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('indexed ordinary/archive pages preserve all13tables and original source indexes/legacy offsets', async () => { + const { source, k, userId } = await fixture() + const legacy = async () => { + const all: Record>> = {} + await runInSeries( + [ + ['findTransactions', 'transactionId'], + ['findOutputs', 'outputId'], + ['findCertificates', 'certificateId'], + ['findTxLabels', 'txLabelId'], + ['findOutputBaskets', 'basketId'], + ['findOutputTags', 'outputTagId'], + ['findCommissions', 'commissionId'], + ['findSyncStates', 'syncStateId'] + ] as const, + async ([method, key]) => { + all[method] = [] + await runInSeries([0, 1], async offset => { + const rows = await source[method]({ partial: { userId }, paged: { limit: 1, offset } }) + all[method].push(rows.map(row => (row as unknown as Record)[key])) + }) + } + ) + all.certificateFields = [] + await runInSeries([0, 1], async offset => { + const rows = await source.findCertificateFields({ partial: { userId }, paged: { limit: 1, offset } }) + all.certificateFields.push(rows.flatMap(row => [row.fieldName, row.certificateId])) + }) + return all + } + const standard = async () => + await k('sqlite_master') + .whereIn('type', ['table', 'index']) + .whereNotIn('tbl_name', ['snapshot_certificate_field_keys', 'snapshot_certificate_index_progress']) + .select('type', 'name', 'tbl_name', 'sql') + .orderBy('name') + const originalIndexes = await standard() + const originalOffsets = await legacy() + const old = await source.openWalletReadSnapshot(identity) + let baseline + try { + baseline = await pages(old) + } finally { + await old.close() + } + await addSnapshotCertificateIndexes(k) + expect(await standard()).toEqual(originalIndexes) + expect(await legacy()).toEqual(originalOffsets) + const queries: Array<{ sql: string; bindings: Knex.RawBinding[] }> = [] + const listen = (query: { sql: string; bindings: Knex.RawBinding[] }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_certificate_field_keys') + ) + queries.push(query) + } + k.on('query', listen) + try { + const partial = await source.openWalletReadSnapshot(identity) + try { + expect(await pages(partial)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await partial.close() + } + await journal(k) + await runInSeries( + [() => source.openWalletReadSnapshot(identity), () => openKnexSnapshotArchiveSource(source, identity)], + async open => { + const view = await open() + try { + expect(await pages(view)).toEqual(baseline) + } finally { + await view.close() + } + } + ) + expect(queries.length).toBeGreaterThan(0) + } finally { + k.off('query', listen) + } + // Explain only page queries; closure may legitimately visit its certificateal parents. + await runInSeries( + queries.filter(query => query.sql.includes('__snapshotBytes') || query.sql.includes('.*')), + async query => { + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + query.sql, query.bindings) + expect(plan.some(row => row.detail.includes('SEARCH snapshot_certificate_field_keys USING COVERING INDEX'))).toBe( + true + ) + expect(plan.some(row => /SCAN |TEMP B-TREE/.test(row.detail))).toBe(false) + } + ) +}) + +test.each(['ordinary', 'archive'] as const)( + '%s mode and all13table pages stay bound across independent journal/profile/progress writes', + async kind => { + const { source, k, writer, userId, otherId } = await fixture() + await addSnapshotCertificateIndexes(k) + const open = async () => + kind === 'ordinary' + ? await source.openWalletReadSnapshot(identity) + : await openKnexSnapshotArchiveSource(source, identity) + const queries: string[] = [] + const listen = (query: { sql: string }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_certificate_field_keys') + ) + queries.push(query.sql) + } + k.on('query', listen) + const old = await open() + let baseline + try { + baseline = await pages(old) + queries.length = 0 + await journal(writer) + expect(await pages(old)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await old.close() + } + const indexed = await open() + try { + await writer.transaction(async trx => { + await trx('snapshot_certificate_index_progress').where('snapshotTableId', 0).update({ complete: 0 }) + await trx('certificates').where('certificateId', 3).update({ userId: otherId }) + await trx('certificate_fields').where('certificateId', 3).update({ userId: otherId }) + await trx('certificates').insert({ + ...dates, + certificateId: 4, + userId, + serialNumber: 'new-after-view', + type: 'type', + certifier: identity, + subject: identity, + revocationOutpoint: 'a'.repeat(64) + '.0', + signature: 'synthetic', + isDeleted: false + }) + }) + expect(await pages(indexed)).toEqual(baseline) + expect(queries.length).toBeGreaterThan(0) + } finally { + await indexed.close() + k.off('query', listen) + } + await expect(open()).rejects.toThrow('migration is incomplete') + await writer('snapshot_certificate_index_progress').where('snapshotTableId', 0).update({ complete: 1 }) + const fresh = await open() + try { + expect((await fresh.readPage('certificates')).rows.map(row => row.certificateId)).toEqual([1, 4]) + } finally { + await fresh.close() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.mysql.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.mysql.test.ts new file mode 100644 index 000000000..9711aba9d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.mysql.test.ts @@ -0,0 +1,402 @@ +import { knex } from 'knex' +import { + addSnapshotCertificateIndexes as install, + removeSnapshotCertificateIndexes as remove, + readSnapshotCertificateIndexState as enabled, + SNAPSHOT_CERTIFICATE_INDEX_MIGRATION as migration +} from '../schema/snapshotCertificateIndexMigration' + +type Kind = 'sourceTables' | 'sourceColumn' | 'tables' | 'columns' | 'indexes' | 'triggers' +type Metadata = Array> + +// Actual installed MySQL compiler/transaction protocol. Separate native tests +// prove database locking, collation, crash recovery and row-read behavior. +function fixture(change: (kind: Kind, rows: Metadata) => unknown = (_kind, rows) => rows) { + const k = knex({ client: 'mysql2' }) + const queries: Array<{ sql: string; values: unknown[] }> = [] + const tables = new Set(['knex_migrations']) + const indexes = new Map() + const triggers = new Map>() + let progress: + | { + snapshotTableId: number + started: boolean + afterFieldName: string + afterCertificateId: number + complete: boolean + } + | undefined + const keys: Array<{ + snapshotUserId: number + snapshotFieldName: string + snapshotCertificateId: number + snapshotMembership: number + }> = [] + let journaled = false + const answer = (sql: string, values: unknown[]): unknown => { + queries.push({ sql, values }) + if (sql.startsWith('SELECT TABLE_NAME AS name')) { + expect(values).toEqual(['certificate_fields', 'certificates']) + return change('sourceTables', [ + { name: 'certificate_fields', engine: 'InnoDB' }, + { name: 'certificates', engine: 'InnoDB' } + ]) + } + if (sql.startsWith('SELECT COLUMN_TYPE AS type')) { + expect(values).toEqual(['certificate_fields', 'fieldName']) + return change('sourceColumn', [ + { type: 'varchar(100)', nullable: 'NO', charset: 'utf8mb4', collation: 'utf8mb4_0900_ai_ci' } + ]) + } + if (sql.startsWith('SELECT ENGINE AS engine')) return change('tables', [{ engine: 'InnoDB' }]) + if (sql.startsWith('select * from information_schema.tables')) + return tables.has(String(values[0])) ? [{ TABLE_NAME: values[0] }] : [] + if (sql.startsWith('create table')) { + tables.add(sql.match(/^create table `([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('drop table')) { + tables.delete(sql.match(/`([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('SELECT COLUMN_NAME')) { + const isKeys = values[0] === 'snapshot_certificate_field_keys' + const names = isKeys + ? ['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId', 'snapshotMembership'] + : ['snapshotTableId', 'started', 'afterFieldName', 'afterCertificateId', 'complete'] + const types = isKeys + ? ['int unsigned', 'varchar(100)', 'int unsigned', 'int unsigned'] + : ['int', 'tinyint', 'varchar(100)', 'int unsigned', 'tinyint'] + return change( + 'columns', + names.map((name, i) => ({ + name, + type: types[i], + nullable: 'NO', + defaultValue: null, + extra: '', + charset: types[i] === 'varchar(100)' ? 'utf8mb4' : null, + collation: types[i] === 'varchar(100)' ? 'utf8mb4_0900_ai_ci' : null + })) + ) + } + if (sql.startsWith('alter table `snapshot_certificate_field_keys` add index')) { + indexes.set( + sql.match(/add index `([^`]+)`/)![1], + [...sql.matchAll(/`([^`]+)`/g)].slice(2).map(match => match[1]) + ) + return [] + } + if (sql.startsWith('SELECT INDEX_NAME FROM')) + return indexes.has(String(values[1])) ? [{ INDEX_NAME: values[1] }] : [] + if (sql.startsWith('SELECT INDEX_NAME AS')) { + const isKeys = values[0] === 'snapshot_certificate_field_keys' + const primary = isKeys ? ['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId'] : ['snapshotTableId'] + return change('indexes', [ + ...primary.map(columnName => ({ name: 'PRIMARY', columnName, nonUnique: 0, direction: 'A', prefix: null })), + ...(isKeys + ? [...indexes].flatMap(([name, columns]) => + columns.map(columnName => ({ name, columnName, nonUnique: 1, direction: 'A', prefix: null })) + ) + : []) + ]) + } + if (sql.startsWith('SELECT EVENT_MANIPULATION')) + return change('triggers', triggers.has(String(values[0])) ? [triggers.get(String(values[0]))!] : []) + if (sql.startsWith('CREATE TRIGGER')) { + const [, name, timing, event, tableName, body] = sql.match( + /^CREATE TRIGGER (\w+) (BEFORE|AFTER) (\w+) ON (\w+) FOR EACH ROW (.*)$/ + )! + triggers.set(name, { event, timing, tableName, body }) + return [] + } + if (sql.startsWith('DROP TRIGGER')) { + triggers.delete(sql.split(' ').at(-1)!.replaceAll('`', '')) + return [] + } + if (sql.startsWith('insert ignore into `snapshot_certificate_index_progress`')) { + const [afterCertificateId, afterFieldName, complete, snapshotTableId, started] = values as [ + number, + string, + boolean, + number, + boolean + ] + progress ??= { snapshotTableId, started, afterFieldName, afterCertificateId, complete } + return { affectedRows: 1, insertId: 0 } + } + if (sql.startsWith('select * from `snapshot_certificate_index_progress`')) { + if (sql.includes('where')) expect(sql).toMatch(/for update$/) + else expect(values).toEqual([2]) + return progress === undefined ? [] : [{ ...progress }] + } + if (sql.startsWith('update `snapshot_certificate_index_progress`')) { + const [started, afterFieldName, afterCertificateId, complete, tableId] = values as [ + boolean, + string, + number, + boolean, + number + ] + expect(tableId).toBe(0) + Object.assign(progress!, { started, afterFieldName, afterCertificateId, complete }) + return { affectedRows: 1 } + } + if (sql.startsWith('insert into `snapshot_certificate_field_keys`')) { + const fields = sql + .slice(sql.indexOf('(') + 1, sql.indexOf(')')) + .split(', ') + .map(value => value.replaceAll('`', '')) + for (let i = 0; i < values.length - 1; i += fields.length) { + const row = Object.fromEntries( + fields.map((field, offset) => [field, values[i + offset]]) + ) as (typeof keys)[number] + const found = keys.find( + key => + key.snapshotUserId === row.snapshotUserId && + key.snapshotFieldName === row.snapshotFieldName && + key.snapshotCertificateId === row.snapshotCertificateId + ) + if (found === undefined) keys.push(row) + else found.snapshotMembership |= Number(values.at(-1)) + } + return { affectedRows: 1 } + } + if (sql.startsWith('select `name` from `knex_migrations`')) return journaled ? [{ name: migration }] : [] + if (/^(BEGIN|COMMIT|ROLLBACK)/.test(sql)) return [] + if (sql.startsWith('select `userId`, `fieldName`, `certificateId` from `certificate_fields`')) { + expect(sql).toMatch(/for update$/) + expect(values.at(-1)).toBe(256) + const rows = Array.from({ length: 257 }, (_, i) => ({ + userId: 1, + fieldName: String(i).padStart(6, '0'), + certificateId: i + 1 + })) + if (values.length === 1) return rows.slice(0, 256) + expect(sql).toContain('(fieldName > ? OR (fieldName = ? AND certificateId > ?))') + return rows + .filter( + row => + row.fieldName > String(values[0]) || (row.fieldName === values[1] && row.certificateId > Number(values[2])) + ) + .slice(0, 256) + } + if (sql.startsWith('select `certificateId`, `userId` from `certificates`')) { + expect(sql).toMatch(/lock in share mode$/) + return values.map(certificateId => ({ certificateId, userId: (Number(certificateId) % 2) + 1 })) + } + throw new Error('Unexpected synthetic driver query: ' + sql) + } + const connection = { + query( + query: { sql: string }, + values: unknown[], + callback: (error: Error | null, rows?: unknown, fields?: unknown[]) => void + ) { + try { + callback(null, answer(query.sql, values), []) + } catch (error) { + callback(error as Error) + } + } + } + jest.spyOn(k.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(k.client, 'releaseConnection').mockResolvedValue(undefined) + return { + k, + queries, + tables, + indexes, + triggers, + keys, + setJournaled: () => { + journaled = true + }, + progress: () => progress + } +} +afterEach(() => jest.restoreAllMocks()) + +test('MySQL owns transactional exact-collation tables, installs observers first, and locks two bootstrap batches', async () => { + const f = fixture() + try { + expect(await enabled(f.k)).toBe(false) + await install(f.k) + await install(f.k) + expect(f.triggers.size).toBe(8) + const ddl = f.queries.filter(q => q.sql.startsWith('CREATE TRIGGER')).map(q => q.sql) + expect(ddl.slice(0, 4).every(sql => sql.includes('DELETE') || sql.includes('BEFORE UPDATE'))).toBe(true) + expect(ddl[4]).toContain('snapshot_certificate_field_insert') + expect(ddl.some(sql => sql.includes('CAST(OLD.fieldName AS BINARY)'))).toBe(true) + expect(ddl.filter(sql => sql.includes('FOR SHARE'))).toHaveLength(4) + expect( + ddl.filter(sql => + sql.includes( + 'ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | ' + ) + ) + ).toHaveLength(4) + expect(f.queries.filter(q => q.sql.startsWith('create table')).every(q => q.sql.includes('engine = InnoDB'))).toBe( + true + ) + expect( + f.queries + .filter(q => q.sql.startsWith('create table')) + .every(q => q.sql.includes('varchar(100) CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci')) + ).toBe(true) + expect(f.progress()).toEqual({ + snapshotTableId: 0, + started: true, + afterFieldName: '000256', + afterCertificateId: 257, + complete: true + }) + expect(f.keys).toHaveLength(386) + expect(f.keys.find(key => key.snapshotFieldName === '000001')).toEqual({ + snapshotCertificateId: 2, + snapshotFieldName: '000001', + snapshotMembership: 3, + snapshotUserId: 1 + }) + expect(await enabled(f.k)).toBe(false) + f.setJournaled() + expect(await enabled(f.k)).toBe(true) + await remove(f.k) + await remove(f.k) + expect(f.tables).toEqual(new Set(['knex_migrations'])) + expect(f.triggers.size).toBe(0) + } finally { + await f.k.destroy() + } +}) + +test.each([ + ['sourceTables', []], + [ + 'sourceTables', + [ + { name: 'certificate_fields', engine: 'MyISAM' }, + { name: 'certificates', engine: 'InnoDB' } + ] + ], + ['sourceColumn', []], + ['sourceColumn', [{ type: 'text', nullable: 'NO', charset: 'utf8mb4', collation: 'utf8mb4_bin' }]], + ['sourceColumn', [{ type: 'varchar(100)', nullable: 'YES', charset: 'utf8mb4', collation: 'utf8mb4_bin' }]], + ['sourceColumn', [{ type: 'varchar(100)', nullable: 'NO', charset: 'invalid charset', collation: 'utf8mb4_bin' }]], + ['sourceColumn', [{ type: 'varchar(100)', nullable: 'NO', charset: 'utf8mb4', collation: 'invalid collation' }]], + ['tables', []], + ['tables', [{ engine: 'MyISAM' }]], + ['columns', []], + ['columns', null], + ['indexes', []], + ['indexes', null] +] as Array<[Kind, unknown]>)('invalid %s metadata refuses adoption and removal', async (kind, value) => { + const f = fixture((current, rows) => (current === kind ? value : rows)) + const message = + kind === 'sourceTables' + ? 'Snapshot certificate source requires transactional tables' + : kind === 'sourceColumn' + ? 'Unsupported snapshot certificate field definition' + : 'Snapshot certificate table definition mismatch' + try { + await expect(install(f.k)).rejects.toThrow(message) + await expect(remove(f.k)).rejects.toThrow(message) + } finally { + await f.k.destroy() + } +}) + +test('MySQL resumes equivalent trigger whitespace without replacing the installed observers', async () => { + let reformatted = false + const f = fixture((kind, rows) => + reformatted && kind === 'triggers' + ? rows.map(row => ({ ...row, body: ' \n' + String(row.body).replaceAll(' ', '\n\t ') + '\n ' })) + : rows + ) + try { + await install(f.k) + const created = f.queries.filter(q => q.sql.startsWith('CREATE TRIGGER')).length + reformatted = true + await install(f.k) + expect(f.queries.filter(q => q.sql.startsWith('CREATE TRIGGER'))).toHaveLength(created) + await remove(f.k) + expect(f.triggers.size).toBe(0) + } finally { + await f.k.destroy() + } +}) + +test.each([ + { name: 'foreign' }, + { type: 'bigint unsigned' }, + { nullable: 'YES' }, + { defaultValue: 0 }, + { extra: 'auto_increment' }, + { charset: 'utf8mb4' }, + { collation: 'utf8mb4_bin' } +])('mismatched column %j cannot be adopted', async changed => { + const f = fixture((kind, rows) => (kind === 'columns' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows)) + try { + await expect(install(f.k)).rejects.toThrow('table definition mismatch') + } finally { + await f.k.destroy() + } +}) + +test.each([{ columnName: 'wrong' }, { direction: 'D' }, { prefix: 10 }, { name: 'foreign_unique' }])( + 'mismatched index %j cannot be adopted', + async changed => { + const f = fixture((kind, rows) => (kind === 'indexes' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows)) + try { + await expect(install(f.k)).rejects.toThrow('table definition mismatch') + } finally { + await f.k.destroy() + } + } +) + +test.each([{ event: 'DELETE' }, { timing: 'BEFORE' }, { tableName: 'other' }, { body: 'BEGIN SELECT 1; END' }])( + 'mismatched trigger %j refuses before dropping any trigger', + async changed => { + let corrupt = false + const f = fixture((kind, rows) => + corrupt && kind === 'triggers' && rows.length ? [{ ...rows[0], ...changed }] : rows + ) + try { + await install(f.k) + corrupt = true + await expect(install(f.k)).rejects.toThrow('trigger definition mismatch') + await expect(remove(f.k)).rejects.toThrow('trigger definition mismatch') + expect(f.triggers.size).toBe(8) + } finally { + await f.k.destroy() + } + } +) + +test.each([ + null, + [ + { event: 'INSERT', timing: 'AFTER', tableName: 'certificate_fields', body: 'BEGIN END' }, + { event: 'INSERT', timing: 'AFTER', tableName: 'certificate_fields', body: 'BEGIN END' } + ] +])('malformed trigger metadata refuses: %j', async value => { + const f = fixture((kind, rows) => (kind === 'triggers' ? value : rows)) + try { + await expect(install(f.k)).rejects.toThrow('trigger') + } finally { + await f.k.destroy() + } +}) + +test('MySQL refuses nesting migration transactions before inspecting or changing objects', async () => { + const f = fixture() + try { + Object.defineProperty(f.k, 'isTransaction', { value: true }) + await expect(install(f.k)).rejects.toThrow('independent DDL') + await expect(remove(f.k)).rejects.toThrow('independent DDL') + expect(f.queries).toEqual([]) + } finally { + await f.k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.test.ts new file mode 100644 index 000000000..e42dedd7b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCertificateIndexes.test.ts @@ -0,0 +1,355 @@ +import type { Knex } from 'knex' +import fc from 'fast-check' +import { runInSeries } from '../../utility/runInSeries' +import { + minimalCertificateDatabase, + expectCertificateMembership, + seedCertificateFields, + applyCertificateOperation, + certificateFieldNames +} from '../../../test/utils/snapshotCertificateFixtures' +import { + addSnapshotCertificateIndexes as install, + removeSnapshotCertificateIndexes as remove, + readSnapshotCertificateIndexState as enabled, + SNAPSHOT_CERTIFICATE_INDEX_MIGRATION as migration +} from '../schema/snapshotCertificateIndexMigration' + +const databases: Knex[] = [] +async function fixture(collation = 'BINARY'): Promise { + const k = await minimalCertificateDatabase(collation) + databases.push(k) + return k +} +afterEach(async () => { + jest.restoreAllMocks() + await runInSeries(databases.splice(0), k => k.destroy()) +}) + +async function journal(k: Knex, table = 'knex_migrations'): Promise { + await k.schema.createTable(table, t => { + t.string('name') + }) + await k(table).insert({ name: migration }) +} + +test.each([0, 1, 255, 256, 257])( + 'bootstrap preserves the empty-name key for %s rows, source schema and repeat install/down', + async count => { + const k = await fixture() + if (count) { + await k('certificates').insert( + Array.from({ length: count }, (_, i) => ({ certificateId: i + 1, userId: (i % 2) + 1 })) + ) + await k('certificate_fields').insert( + Array.from({ length: count }, (_, i) => ({ + certificateId: i + 1, + userId: 1, + fieldName: '', + fieldValue: 'value' + })) + ) + } + const schema = () => + k('sqlite_master').whereIn('type', ['table', 'index']).select('name', 'type', 'sql').orderBy('name') + const before = await schema() + await install(k) + await install(k) + await expectCertificateMembership(k) + expect(await k('snapshot_certificate_index_progress').first()).toEqual({ + snapshotTableId: 0, + started: count ? 1 : 0, + afterFieldName: '', + afterCertificateId: count, + complete: 1 + }) + await remove(k) + await remove(k) + expect(await schema()).toEqual(before) + expect(await k('certificate_fields')).toHaveLength(count) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) + } +) + +test.each(['BINARY', 'NOCASE', 'RTRIM'])( + 'generated ownership, rekey and rollback schedules preserve source collation %s', + async collation => { + const k = await fixture(collation) + await seedCertificateFields(k) + await install(k) + await expectCertificateMembership(k) + const operation = fc.record({ + kind: fc.integer({ min: 0, max: 8 }), + id: fc.integer({ min: 1, max: 4 }), + user: fc.integer({ min: 1, max: 3 }), + name: fc.integer({ min: 0, max: certificateFieldNames.length - 1 }) + }) + await fc.assert( + fc.asyncProperty(fc.array(operation, { minLength: 1, maxLength: 24 }), async schedule => { + await seedCertificateFields(k) + await runInSeries(schedule, async op => { + await applyCertificateOperation(k, op) + await expectCertificateMembership(k) + }) + }), + { numRuns: 300, seed: 3242026 } + ) + } +) + +test('journal, complete positions and exact tables gate adoption in a custom migration journal', async () => { + const k = await fixture() + expect(await enabled(k)).toBe(false) + await install(k) + expect(await enabled(k)).toBe(false) + await journal(k, 'custom_journal') + const config = { tableName: 'custom_journal', schemaName: 'main' } + expect(await enabled(k, config)).toBe(true) + await k('custom_journal').update({ name: 'unrelated migration' }) + expect(await enabled(k, config)).toBe(false) + await k('custom_journal').update({ name: migration }) + await k('snapshot_certificate_index_progress').update({ complete: 0 }) + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') + await k('snapshot_certificate_index_progress').update({ complete: 1 }) + await k('snapshot_certificate_index_progress').insert({ + snapshotTableId: 1, + started: 0, + afterFieldName: '', + afterCertificateId: 0, + complete: 1 + }) + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') + await k.schema.dropTable('snapshot_certificate_index_progress') + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') + await k.schema.dropTable('snapshot_certificate_field_keys') + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') +}) + +test.each([ + { started: 2 }, + { complete: 2 }, + { afterCertificateId: -1 }, + { afterCertificateId: 1.5 }, + { afterCertificateId: Number.MAX_SAFE_INTEGER + 1 }, + { afterCertificateId: 'invalid' }, + { afterFieldName: 'x'.repeat(101) }, + { afterFieldName: 'x' }, + { afterCertificateId: 1 }, + { started: 1, afterCertificateId: 0 } +])('invalid progress %j refuses before source reads', async change => { + const k = await fixture() + await install(k) + await k('snapshot_certificate_index_progress').update({ complete: 0, ...change }) + const query = jest.fn() + k.on('query', query) + await expect(install(k)).rejects.toThrow('bootstrap position') + expect(query.mock.calls.some(([q]) => q.sql.startsWith('select `userId`, `fieldName`, `certificateId`'))).toBe(false) +}) + +test.each([ + ['userId', 0], + ['userId', -1], + ['userId', 1.5], + ['userId', 'invalid'], + ['certificateId', 0], + ['certificateId', -1], + ['certificateId', 1.5], + ['fieldName', 'x'.repeat(101)] +])('invalid source %s=%s rolls back keys and cursor', async (field, value) => { + const k = await fixture() + await k('certificate_fields').insert({ + userId: 1, + fieldName: 'a', + certificateId: 1, + fieldValue: 'v', + [field as string]: value + }) + await expect(install(k)).rejects.toThrow('source key') + expect(await k('snapshot_certificate_field_keys')).toEqual([]) + expect(await k('snapshot_certificate_index_progress').first()).toMatchObject({ + started: 0, + complete: 0, + afterCertificateId: 0 + }) +}) + +test('committed bootstrap resumes behind-cursor changes after an interrupted next batch', async () => { + const k = await fixture() + await k('certificates').insert({ certificateId: 1, userId: 1 }) + await k('certificate_fields').insert( + Array.from({ length: 300 }, (_, i) => ({ + userId: 1, + certificateId: 1, + fieldName: String(i + 1).padStart(6, '0'), + fieldValue: 'v' + })) + ) + const failure = new Error('synthetic certificate migration interruption') + let reads = 0 + const interrupt = (q: { sql: string }): void => { + if (q.sql.startsWith('select `userId`, `fieldName`, `certificateId`') && ++reads === 2) throw failure + } + k.on('query', interrupt) + try { + await expect(install(k)).rejects.toBe(failure) + } finally { + k.off('query', interrupt) + } + expect(await k('snapshot_certificate_index_progress').first()).toMatchObject({ + started: 1, + afterFieldName: '000256', + afterCertificateId: 1, + complete: 0 + }) + expect(await k('snapshot_certificate_field_keys')).toHaveLength(256) + await k('certificate_fields').insert({ userId: 2, certificateId: 1, fieldName: '', fieldValue: 'late' }) + await k('certificates').update({ userId: 3 }) + await k('certificate_fields').where('fieldName', '000001').delete() + await install(k) + await expectCertificateMembership(k) +}) + +test('changed triggers refuse install and removal before altering any source or trigger', async () => { + const k = await fixture() + await install(k) + await k.raw('DROP TRIGGER snapshot_certificate_field_insert') + await k.raw('CREATE TRIGGER snapshot_certificate_field_insert AFTER INSERT ON certificate_fields BEGIN SELECT 1; END') + const before = await k('sqlite_master').where('type', 'trigger').orderBy('name') + await expect(install(k)).rejects.toThrow('trigger definition mismatch') + await expect(remove(k)).rejects.toThrow('trigger definition mismatch') + expect(await k('sqlite_master').where('type', 'trigger').orderBy('name')).toEqual(before) +}) + +test.each([ + 'CREATE UNIQUE INDEX foreign_certificate ON snapshot_certificate_field_keys(snapshotCertificateId)', + 'ALTER TABLE snapshot_certificate_field_keys ADD COLUMN unexpected INTEGER', + 'DROP INDEX snapshot_certificate_parent' +])('schema inspection detects incompatible definitions: %s', async sql => { + const k = await fixture() + await install(k) + await journal(k) + await k.raw(sql) + await expect(enabled(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') + if (sql.startsWith('DROP')) { + await install(k) + expect(await enabled(k)).toBe(true) + } else await expect(install(k)).rejects.toThrow('table definition mismatch') +}) + +test.each([ + 'snapshotCertificateId DESC,snapshotUserId,snapshotFieldName', + 'snapshotCertificateId,snapshotUserId,snapshotFieldName COLLATE NOCASE' +])('incompatible maintenance index %s refuses adoption', async columns => { + const k = await fixture() + await install(k) + await k.raw('DROP INDEX snapshot_certificate_parent') + await k.raw('CREATE INDEX snapshot_certificate_parent ON snapshot_certificate_field_keys(' + columns + ')') + await expect(install(k)).rejects.toThrow('table definition mismatch') +}) + +test('a separately collated unique index cannot redefine the source cursor order', async () => { + const k = await fixture() + await k.schema.dropTable('certificate_fields') + await k.raw( + 'CREATE TABLE certificate_fields(userId INTEGER,fieldName VARCHAR(100),certificateId INTEGER,fieldValue TEXT)' + ) + await k.raw('CREATE UNIQUE INDEX wrong_order ON certificate_fields(fieldName COLLATE NOCASE,certificateId)') + await expect(install(k)).rejects.toThrow('field order') + expect(await k.schema.hasTable('snapshot_certificate_field_keys')).toBe(false) +}) + +test.each([ + 'fieldName', + 'certificateId,fieldName', + 'fieldName,certificateId,userId', + 'fieldName DESC,certificateId', + 'fieldName,certificateId DESC', + 'fieldName,certificateId COLLATE NOCASE' +])('an incompatible source unique key %s cannot define the snapshot order', async columns => { + const k = await fixture() + await k.schema.dropTable('certificate_fields') + await k.raw( + 'CREATE TABLE certificate_fields(userId INTEGER,fieldName VARCHAR(100),certificateId INTEGER,fieldValue TEXT)' + ) + await k.raw('CREATE UNIQUE INDEX unsupported_source ON certificate_fields(' + columns + ')') + await expect(install(k)).rejects.toThrow('Unsupported snapshot certificate field order') + expect(await k.schema.hasTable('snapshot_certificate_field_keys')).toBe(false) +}) + +test.each([ + 'renamedMembership INTEGER NOT NULL', + 'snapshotMembership TEXT NOT NULL', + 'snapshotMembership INTEGER', + 'snapshotMembership INTEGER NOT NULL DEFAULT 0', + 'snapshotMembership INTEGER GENERATED ALWAYS AS (1) VIRTUAL NOT NULL' +])('an incompatible owned column %s cannot be resumed, adopted or removed', async membership => { + const k = await fixture() + await install(k) + await journal(k) + await k.schema.dropTable('snapshot_certificate_field_keys') + await k.raw( + 'CREATE TABLE snapshot_certificate_field_keys(snapshotUserId INTEGER NOT NULL,snapshotFieldName VARCHAR(100) NOT NULL,snapshotCertificateId INTEGER NOT NULL,' + + membership + + ',PRIMARY KEY(snapshotUserId,snapshotFieldName,snapshotCertificateId))' + ) + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(enabled(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') + expect(await k.schema.hasTable('snapshot_certificate_field_keys')).toBe(true) +}) + +test('resume accepts the full-width field cursor and refuses an unrelated progress row', async () => { + const k = await fixture() + await k('certificate_fields').insert({ userId: 1, certificateId: 1, fieldName: '😀'.repeat(100), fieldValue: 'v' }) + await install(k) + await journal(k) + expect(await enabled(k)).toBe(true) + await install(k) + await expectCertificateMembership(k) + await k('snapshot_certificate_index_progress').update({ snapshotTableId: 1 }) + await expect(enabled(k)).rejects.toThrow('migration is incomplete') +}) + +test('bootstrap retains orphan fields as directly owned without inventing parent membership', async () => { + const k = await fixture() + await k('certificate_fields').insert({ userId: 7, fieldName: 'orphan', certificateId: 99, fieldValue: 'synthetic' }) + await install(k) + await expectCertificateMembership(k) + expect(await k('snapshot_certificate_field_keys')).toEqual([ + { snapshotUserId: 7, snapshotFieldName: 'orphan', snapshotCertificateId: 99, snapshotMembership: 1 } + ]) +}) + +test.each([ + 'fieldName', + 'certificateId,fieldName', + 'fieldName,certificateId DESC', + 'fieldName,certificateId COLLATE NOCASE' +])('incompatible source unique index %s refuses before installing objects', async columns => { + const k = await fixture() + await k.schema.dropTable('certificate_fields') + await k.raw( + 'CREATE TABLE certificate_fields(userId INTEGER,fieldName VARCHAR(100),certificateId INTEGER,fieldValue TEXT)' + ) + await k.raw('CREATE UNIQUE INDEX wrong_order ON certificate_fields(' + columns + ')') + await k.raw('CREATE INDEX nonunique_order ON certificate_fields(fieldName,certificateId)') + await k.raw('CREATE UNIQUE INDEX partial_order ON certificate_fields(fieldName,certificateId) WHERE userId=1') + await expect(install(k)).rejects.toThrow('field order') + expect(await k.schema.hasTable('snapshot_certificate_field_keys')).toBe(false) +}) + +test('missing auxiliary primary-index metadata refuses without publishing progress', async () => { + const k = await fixture() + const omit = (rows: unknown, query: { sql: string }): void => { + if (query.sql.startsWith('PRAGMA index_list(') && query.sql.includes('snapshot_certificate_field_keys')) + (rows as unknown[]).splice(0) + } + k.on('query-response', omit) + try { + await expect(install(k)).rejects.toThrow('table definition mismatch') + } finally { + k.off('query-response', omit) + } + expect(await k.schema.hasTable('snapshot_certificate_index_progress')).toBe(false) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCursor.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCursor.test.ts new file mode 100644 index 000000000..b49febc21 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCursor.test.ts @@ -0,0 +1,84 @@ +import { copySnapshotArchivePosition, copySnapshotCursor, sameSnapshotArchivePosition } from './SnapshotCursor' +import type { WalletSnapshotArchivePosition, WalletSnapshotCursor } from './WalletReadSnapshot' + +const position: WalletSnapshotArchivePosition = { version: 1, archiveId: 'a'.repeat(64), sequence: 0, rowOffset: 1 } +const legacy: WalletSnapshotCursor = { version: 1, snapshotId: 'b'.repeat(64), table: 'txLabels', after: [1] } + +test('legacy cursor JSON stays byte-identical while archive metadata and keys are detached', () => { + expect(copySnapshotCursor(undefined)).toBeUndefined() + const copiedLegacy = copySnapshotCursor(legacy) + expect(JSON.stringify(copiedLegacy)).toBe(JSON.stringify(legacy)) + expect(copiedLegacy).not.toHaveProperty('archivePosition') + expect(copiedLegacy.after).not.toBe(legacy.after) + const current = { ...legacy, archivePosition: position } + const copy = copySnapshotCursor(current) + expect(copy).toEqual(current) + expect(copy.archivePosition).not.toBe(position) + expect(copy.after).not.toBe(current.after) + expect(Object.keys(copy.archivePosition!)).toEqual(['version', 'archiveId', 'sequence', 'rowOffset']) + expect(copySnapshotArchivePosition({ ...position, sequence: 4095, rowOffset: 1000 })).toEqual({ + ...position, + sequence: 4095, + rowOffset: 1000 + }) +}) + +test.each([ + { version: 2 }, + { archiveId: '' }, + { archiveId: 'A'.repeat(64) }, + { archiveId: 'g'.repeat(64) }, + { archiveId: 'a'.repeat(63) }, + { archiveId: 'a'.repeat(65) }, + { archiveId: 1 }, + { sequence: -1 }, + { sequence: 4096 }, + { sequence: 1.5 }, + { sequence: NaN }, + { sequence: '1' }, + { rowOffset: 0 }, + { rowOffset: 1001 }, + { rowOffset: 1.5 }, + { rowOffset: NaN }, + { rowOffset: '1' }, + { extra: true }, + { [Symbol('extra')]: true } +])('position accepts only bounded exact metadata %p', change => { + expect(() => copySnapshotArchivePosition({ ...position, ...change })).toThrow('bounded version-one archive position') +}) + +test.each([undefined, null, [], 'position', 1, true])('rejects non-record positions %p', value => { + expect(() => copySnapshotArchivePosition(value)).toThrow('bounded version-one archive position') +}) + +test('positions require own enumerable data without invoking getters', () => { + for (const name of Object.keys(position)) { + const value = { ...position } as Record + delete value[name] + expect(() => copySnapshotArchivePosition(value)).toThrow() + Object.defineProperty(value, name, { value: Reflect.get(position, name), configurable: true }) + expect(() => copySnapshotArchivePosition(value)).toThrow() + Object.defineProperty(value, name, { + enumerable: true, + get: () => { + throw new Error('Getter must not run') + } + }) + expect(() => copySnapshotArchivePosition(value)).toThrow('bounded version-one archive position') + } + expect(() => copySnapshotArchivePosition(Object.create(position))).toThrow() +}) + +test('acknowledgements compare every archive-position binding and distinguish missing metadata', () => { + expect(sameSnapshotArchivePosition(undefined, undefined)).toBe(true) + expect(sameSnapshotArchivePosition(position, undefined)).toBe(false) + expect(sameSnapshotArchivePosition(undefined, position)).toBe(false) + expect(sameSnapshotArchivePosition(position, { ...position })).toBe(true) + expect( + sameSnapshotArchivePosition(position, { rowOffset: 1, sequence: 0, archiveId: 'a'.repeat(64), version: 1 }) + ).toBe(true) + for (const change of [{ version: 2 }, { archiveId: 'b'.repeat(64) }, { sequence: 1 }, { rowOffset: 2 }]) + expect(sameSnapshotArchivePosition(position, { ...position, ...change } as WalletSnapshotArchivePosition)).toBe( + false + ) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCursor.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCursor.ts new file mode 100644 index 000000000..cf4f6c6b5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotCursor.ts @@ -0,0 +1,68 @@ +import { WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +import type { WalletSnapshotArchivePosition, WalletSnapshotCursor } from './WalletReadSnapshot' + +function invalid(): never { + throw new WERR_INVALID_PARAMETER('cursor.archivePosition', 'a bounded version-one archive position') +} + +/** Validates additive metadata without changing the shape of legacy cursors. */ +export function copySnapshotArchivePosition(input: unknown): WalletSnapshotArchivePosition { + const names = ['version', 'archiveId', 'sequence', 'rowOffset'] + if ( + input === null || + typeof input !== 'object' || + Array.isArray(input) || + Reflect.ownKeys(input).length !== names.length + ) + invalid() + const value: Record = {} + for (const name of names) { + const property = Object.getOwnPropertyDescriptor(input, name) + if (property === undefined || !('value' in property) || !property.enumerable) invalid() + value[name] = property.value + } + if ( + value.version !== 1 || + typeof value.archiveId !== 'string' || + !/^[0-9a-f]{64}$/.test(value.archiveId) || + !Number.isSafeInteger(value.sequence) || + (value.sequence as number) < 0 || + (value.sequence as number) >= 4096 || + !Number.isSafeInteger(value.rowOffset) || + (value.rowOffset as number) < 1 || + (value.rowOffset as number) > 1000 + ) + invalid() + return { + version: 1, + archiveId: value.archiveId, + sequence: value.sequence as number, + rowOffset: value.rowOffset as number + } +} + +export function copySnapshotCursor(cursor: WalletSnapshotCursor): WalletSnapshotCursor +export function copySnapshotCursor(cursor: WalletSnapshotCursor | undefined): WalletSnapshotCursor | undefined +export function copySnapshotCursor(cursor: WalletSnapshotCursor | undefined): WalletSnapshotCursor | undefined { + if (cursor === undefined) return undefined + return { + ...cursor, + after: [...cursor.after], + ...(cursor.archivePosition === undefined + ? {} + : { archivePosition: copySnapshotArchivePosition(cursor.archivePosition) }) + } +} + +export function sameSnapshotArchivePosition( + left: WalletSnapshotArchivePosition | undefined, + right: WalletSnapshotArchivePosition | undefined +): boolean { + if (left === undefined || right === undefined) return left === right + return ( + left.version === right.version && + left.archiveId === right.archiveId && + left.sequence === right.sequence && + left.rowOffset === right.rowOffset + ) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.integration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.integration.test.ts new file mode 100644 index 000000000..96c14f600 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.integration.test.ts @@ -0,0 +1,231 @@ +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { runInSeries } from '../../utility/runInSeries' +import { seedArchiveClosure } from '../../../test/utils/snapshotArchiveFixtures' +import { snapshotArchiveTables } from './archive/SnapshotArchive' +import { openKnexSnapshotArchiveSource, type SnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import { + addSnapshotGlobalIndexes, + removeSnapshotGlobalIndexes, + SNAPSHOT_GLOBAL_INDEX_MIGRATION +} from '../schema/snapshotGlobalIndexMigration' + +const identity = '02' + '11'.repeat(32) +const stores: StorageKnex[] = [] +const writers: Knex[] = [] +const directories: string[] = [] +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-global-index-')) + directories.push(directory) + const options = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + } + const k = knex(options) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + stores.push(source) + await k.raw('PRAGMA journal_mode=WAL') + await migrateBeforeSqliteGeneration(source, 'global index fixture', 'synthetic-global-index') + await source.makeAvailable() + // Start with the immediately preceding schema on both old and new source. + await removeSnapshotGlobalIndexes(k) + await k('knex_migrations').where('name', SNAPSHOT_GLOBAL_INDEX_MIGRATION).delete() + const { user } = await source.findOrInsertUser(identity) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, other.userId) + const writer = knex(options) + writers.push(writer) + return { source, writer, k, userId: user.userId, otherId: other.userId } +} +async function journal(k: Knex): Promise { + await k('knex_migrations').insert({ + name: SNAPSHOT_GLOBAL_INDEX_MIGRATION, + batch: 99, + migration_time: new Date() + }) +} +async function pages(view: WalletReadSnapshot | SnapshotArchiveSource) { + expect(Object.hasOwn(view, 'globalIndexes')).toBe(false) + if ('validateClosure' in view) await view.validateClosure() + const result: Record = {} + await runInSeries(snapshotArchiveTables, async table => { + let cursor: WalletSnapshotCursor | undefined + const selected: unknown[] = [] + let complete = false + for (let pageNumber = 0; pageNumber < 25 && !complete; pageNumber++) { + const page = await view.readPage(table, cursor, { maxRows: 1, maxBytes: 131072 }) + selected.push({ rows: page.rows, after: page.cursor?.after, payloadBytes: page.payloadBytes, done: page.done }) + complete = page.done + cursor = page.cursor + } + expect(complete).toBe(true) + result[table] = selected + }) + return result +} +afterEach(async () => { + await runInSeries(writers.splice(0), k => k.destroy()) + await runInSeries(stores.splice(0), source => source.destroy()) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('indexed ordinary/archive pages preserve all13tables and original source indexes/legacy offsets', async () => { + const { source, k, userId } = await fixture() + const legacy = async () => { + const all: Record>> = {} + await runInSeries( + [ + ['findTransactions', 'transactionId'], + ['findOutputs', 'outputId'], + ['findCertificates', 'certificateId'], + ['findTxLabels', 'txLabelId'], + ['findOutputBaskets', 'basketId'], + ['findOutputTags', 'outputTagId'], + ['findCommissions', 'commissionId'], + ['findSyncStates', 'syncStateId'] + ] as const, + async ([method, key]) => { + all[method] = [] + await runInSeries([0, 1], async offset => { + const rows = await source[method]({ partial: { userId }, paged: { limit: 1, offset } }) + all[method].push(rows.map(row => (row as unknown as Record)[key])) + }) + } + ) + all.certificateFields = [] + await runInSeries([0, 1], async offset => { + const rows = await source.findCertificateFields({ partial: { userId }, paged: { limit: 1, offset } }) + all.certificateFields.push(rows.flatMap(row => [row.fieldName, row.certificateId])) + }) + return all + } + const standard = async () => + await k('sqlite_master') + .whereIn('type', ['table', 'index']) + .whereNotIn('tbl_name', [ + 'snapshot_global_keys', + 'snapshot_global_edges', + 'snapshot_global_guards', + 'snapshot_global_index_progress' + ]) + .select('type', 'name', 'tbl_name', 'sql') + .orderBy('name') + const originalIndexes = await standard() + const originalOffsets = await legacy() + const old = await source.openWalletReadSnapshot(identity) + let baseline + try { + baseline = await pages(old) + } finally { + await old.close() + } + await addSnapshotGlobalIndexes(k) + expect(await standard()).toEqual(originalIndexes) + expect(await legacy()).toEqual(originalOffsets) + const queries: Array<{ sql: string; bindings: Knex.RawBinding[] }> = [] + const listen = (query: { sql: string; bindings: Knex.RawBinding[] }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_global_keys') + ) + queries.push(query) + } + k.on('query', listen) + try { + const partial = await source.openWalletReadSnapshot(identity) + try { + expect(await pages(partial)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await partial.close() + } + await journal(k) + await runInSeries( + [() => source.openWalletReadSnapshot(identity), () => openKnexSnapshotArchiveSource(source, identity)], + async open => { + const view = await open() + try { + expect(await pages(view)).toEqual(baseline) + } finally { + await view.close() + } + } + ) + expect(queries.length).toBeGreaterThan(0) + } finally { + k.off('query', listen) + } + // Explain page queries; complete closure validation also inspects ownership relations. + await runInSeries( + queries.filter(query => query.sql.includes('__snapshotBytes') || query.sql.includes('.*')), + async query => { + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + query.sql, query.bindings) + expect(plan.some(row => /SEARCH snapshot_global_keys USING (COVERING )?INDEX/.test(row.detail))).toBe(true) + expect(plan.some(row => /SCAN |TEMP B-TREE/.test(row.detail))).toBe(false) + } + ) +}) + +test.each(['ordinary', 'archive'] as const)( + '%s mode and all13table pages stay bound across independent journal/profile/progress writes', + async kind => { + const { source, k, writer } = await fixture() + await addSnapshotGlobalIndexes(k) + const open = async () => + kind === 'ordinary' + ? await source.openWalletReadSnapshot(identity) + : await openKnexSnapshotArchiveSource(source, identity) + const queries: string[] = [] + const listen = (query: { sql: string }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_global_keys') + ) + queries.push(query.sql) + } + k.on('query', listen) + const old = await open() + let baseline + try { + baseline = await pages(old) + queries.length = 0 + await journal(writer) + expect(await pages(old)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await old.close() + } + const indexed = await open() + try { + await writer.transaction(async trx => { + await trx('snapshot_global_index_progress').where('id', 0).update({ complete: 0 }) + await trx('transactions').where('transactionId', 1).update({ provenTxId: 2 }) + await trx('proven_tx_reqs').where('provenTxReqId', 1).update({ provenTxId: 2 }) + }) + expect(await pages(indexed)).toEqual(baseline) + expect(queries.length).toBeGreaterThan(0) + } finally { + await indexed.close() + k.off('query', listen) + } + await expect(open()).rejects.toThrow('migration is incomplete') + await writer('snapshot_global_index_progress').where('id', 0).update({ complete: 1 }) + const fresh = await open() + try { + expect((await fresh.readPage('provenTxs')).rows.map(row => row.provenTxId)).toEqual([2, 3]) + expect((await fresh.readPage('provenTxReqs')).rows.map(row => row.provenTxId)).toEqual([2, 3]) + } finally { + await fresh.close() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.mysql.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.mysql.test.ts new file mode 100644 index 000000000..e7e777e89 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.mysql.test.ts @@ -0,0 +1,659 @@ +import { knex } from 'knex' +import { + addSnapshotGlobalIndexes as install, + removeSnapshotGlobalIndexes as remove, + readSnapshotGlobalIndexState as enabled, + SNAPSHOT_GLOBAL_INDEX_MIGRATION as migration +} from '../schema/snapshotGlobalIndexMigration' + +type Kind = 'engine' | 'rules' | 'sourceColumns' | 'columns' | 'indexes' | 'triggers' +type Metadata = Array> +interface Edge { + transactionId: number + requestId: number + tableId: number + rowId: number + userId: number +} + +// Exercise the installed MySQL query compiler and transaction protocol. Native +// fixtures separately establish trigger effects, concurrency and query bounds. +function fixture(change: (kind: Kind, table: string, rows: Metadata) => unknown = (_kind, _table, rows) => rows) { + const k = knex({ client: 'mysql2' }) + const queries: Array<{ sql: string; values: unknown[] }> = [] + const tables = new Set(['knex_migrations']) + const indexes = new Map>() + const triggers = new Map>() + const edges: Edge[] = [] + const guards = new Set() + let lastSourceCursor = -1 + let journaled = false + let progress: { id: number; afterRowId: number; complete: boolean | number } | undefined + const columns: Record> = { + proven_txs: [['provenTxId', 'int unsigned', 'NO']], + proven_tx_reqs: [ + ['provenTxReqId', 'int unsigned', 'NO'], + ['provenTxId', 'int unsigned', 'YES'], + ['txid', 'varchar(64)', 'NO'] + ], + transactions: [ + ['transactionId', 'int unsigned', 'NO'], + ['userId', 'int unsigned', 'NO'], + ['provenTxId', 'int unsigned', 'YES'], + ['txid', 'varchar(64)', 'YES'] + ], + snapshot_global_guards: [ + ['proofId', 'int unsigned', 'NO'], + ['present', 'tinyint(1)', 'NO'] + ], + snapshot_global_keys: [ + ['tableId', 'int', 'NO'], + ['userId', 'int unsigned', 'NO'], + ['rowId', 'int unsigned', 'NO'], + ['refs', 'bigint unsigned', 'NO'], + ['present', 'tinyint(1)', 'NO'] + ], + snapshot_global_edges: [ + ['transactionId', 'int unsigned', 'NO'], + ['requestId', 'int unsigned', 'NO'], + ['tableId', 'int', 'NO'], + ['rowId', 'int unsigned', 'NO'], + ['userId', 'int unsigned', 'NO'] + ], + snapshot_global_index_progress: [ + ['id', 'int', 'NO'], + ['afterRowId', 'int unsigned', 'NO'], + ['complete', 'tinyint(1)', 'NO'] + ] + } + const primary: Record = { + proven_txs: ['provenTxId'], + proven_tx_reqs: ['provenTxReqId'], + transactions: ['transactionId'], + snapshot_global_guards: ['proofId'], + snapshot_global_keys: ['tableId', 'userId', 'rowId'], + snapshot_global_edges: ['transactionId', 'requestId', 'tableId', 'rowId'], + snapshot_global_index_progress: ['id'] + } + const answer = (sql: string, values: unknown[] = []): unknown => { + queries.push({ sql, values }) + const table = String(values[0]) + if (sql.startsWith('SELECT ENGINE AS engine')) return change('engine', table, [{ engine: 'InnoDB' }]) + if (sql.startsWith('SELECT UPDATE_RULE')) + return change('rules', table, [{ updateRule: 'RESTRICT', deleteRule: 'NO ACTION' }]) + if (sql.startsWith('SELECT COLUMN_NAME')) { + const source = !table.startsWith('snapshot_') + return change( + source ? 'sourceColumns' : 'columns', + table, + columns[table].map(([name, type, nullable], i) => ({ + name, + type, + nullable, + defaultValue: null, + extra: source && i === 0 ? 'auto_increment' : '', + charset: type === 'varchar(64)' ? 'utf8mb4' : null, + collation: type === 'varchar(64)' ? 'utf8mb4_0900_ai_ci' : null + })) + ) + } + if (sql.startsWith('SELECT INDEX_NAME AS')) { + const part = (name: string, columnName: string, nonUnique: number) => ({ + name, + columnName, + nonUnique, + direction: 'A', + prefix: null + }) + return change('indexes', table, [ + ...primary[table].map(column => part('PRIMARY', column, 0)), + ...(['transactions', 'proven_tx_reqs'].includes(table) + ? [part('source_txid', 'txid', table === 'transactions' ? 1 : 0)] + : []), + ...[...(indexes.get(table) ?? [])].flatMap(([name, fields]) => fields.map(field => part(name, field, 1))) + ]) + } + if (sql.startsWith('select * from information_schema.tables')) + return tables.has(table) ? [{ TABLE_NAME: table }] : [] + if (sql.startsWith('create table')) { + tables.add(sql.match(/^create table `([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('drop table')) { + tables.delete(sql.match(/`([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('alter table')) { + const [, name, index] = sql.match(/^alter table `([^`]+)` add index `([^`]+)`/)! + const found = indexes.get(name) ?? new Map() + found.set( + index, + [...sql.matchAll(/`([^`]+)`/g)].slice(2).map(match => match[1]) + ) + indexes.set(name, found) + return [] + } + if (sql.startsWith('SELECT EVENT_MANIPULATION')) + return change('triggers', table, triggers.has(table) ? [triggers.get(table)!] : []) + if (sql.startsWith('CREATE TRIGGER')) { + const [, name, timing, event, tableName, body] = sql.match( + /^CREATE TRIGGER (\w+) (BEFORE|AFTER) (\w+) ON (\w+) FOR EACH ROW (.*)$/ + )! + triggers.set(name, { event, timing, tableName, body }) + return [] + } + if (sql.startsWith('DROP TRIGGER')) { + triggers.delete(sql.split(' ').at(-1)!.replaceAll('`', '')) + return [] + } + if (sql.startsWith('insert ignore into `snapshot_global_index_progress`')) { + expect(values).toEqual([0, false, 0]) + progress ??= { id: 0, afterRowId: 0, complete: false } + return { affectedRows: 1, insertId: 0 } + } + if (sql.startsWith('select * from `snapshot_global_index_progress`')) { + if (sql.includes('where')) { + expect(sql).toMatch(/for update$/) + expect(values).toEqual([0, 1]) + } else expect(values).toEqual([2]) + return progress === undefined ? [] : [{ ...progress }] + } + if (sql.startsWith('update `snapshot_global_index_progress`')) { + expect(values[2]).toBe(0) + Object.assign(progress!, { afterRowId: values[0], complete: values[1] }) + return { affectedRows: 1 } + } + if (sql.startsWith('select `transactionId` from `transactions`')) { + expect(sql).toContain('`transactionId` <= ?') + expect(sql).toMatch(/for update$/) + expect(values).toEqual([0, 1]) + return [] + } + if (sql.startsWith('select `transactionId`, `userId`, `provenTxId`')) { + const cursor = Number(values[2]) + expect(cursor).toBeGreaterThan(lastSourceCursor) + lastSourceCursor = cursor + expect(sql).toContain('CASE WHEN octet_length(txid) <= ? THEN txid END AS txid') + expect(sql).toContain('COALESCE(octet_length(txid),?) AS txidBytes') + expect(sql).toContain('where `transactionId` > ? order by `transactionId` asc limit ? for update') + expect(values.slice(0, 2)).toEqual([256, 0]) + expect(values[3]).toBe(256) + return Array.from({ length: 257 }, (_, i) => ({ + transactionId: i + 1, + userId: (i % 2) + 1, + provenTxId: i % 3 ? 7 : null, + txid: i % 2 ? 'a' : null, + txidBytes: i % 2 ? 1 : 0 + })) + .filter(row => row.transactionId > cursor) + .slice(0, Number(values[3])) + } + if (sql.startsWith('select `provenTxReqId`, `provenTxId`')) { + expect(sql).toContain('from `proven_tx_reqs` where `txid` = ? limit ? lock in share mode') + expect(values).toEqual(['a', 1]) + return [{ provenTxReqId: 4294967294, provenTxId: 4294967295 }] + } + if (sql.startsWith('insert into `snapshot_global_guards`')) { + expect(sql).toContain('on duplicate key update `proofId` = `proofId`') + expect(values[0]).toBe(false) + guards.add(Number(values[1])) + return { affectedRows: 1 } + } + if (sql.startsWith('UPDATE snapshot_global_guards')) { + expect(sql).toBe( + 'UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=?' + ) + expect(guards.has(Number(values[0]))).toBe(true) + return { affectedRows: 1 } + } + if (sql.startsWith('insert into `snapshot_global_edges`')) { + expect(sql).toContain('on duplicate key update `transactionId` = `transactionId`') + const fields = sql + .slice(sql.indexOf('(') + 1, sql.indexOf(')')) + .split(', ') + .map(value => value.replaceAll('`', '')) + for (let i = 0; i < values.length; i += fields.length) { + const row = Object.fromEntries(fields.map((field, offset) => [field, values[i + offset]])) as unknown as Edge + if ( + !edges.some( + edge => + edge.transactionId === row.transactionId && + edge.requestId === row.requestId && + edge.tableId === row.tableId && + edge.rowId === row.rowId + ) + ) + edges.push(row) + } + return { affectedRows: 1 } + } + if (sql.startsWith('select `name` from `knex_migrations`')) { + expect(values).toEqual([migration, 1]) + return journaled ? [{ name: migration }] : [] + } + if (/^(BEGIN|COMMIT|ROLLBACK)/.test(sql)) return [] + throw new Error('Unexpected synthetic driver query: ' + sql) + } + const connection = { + query( + query: { sql: string }, + values: unknown[], + callback: (error: Error | null, rows?: unknown, fields?: unknown[]) => void + ) { + try { + callback(null, answer(query.sql, values), []) + } catch (error) { + callback(error as Error) + } + } + } + jest.spyOn(k.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(k.client, 'releaseConnection').mockResolvedValue(undefined) + return { + k, + queries, + tables, + indexes, + triggers, + edges, + progress: () => progress, + setJournaled: () => { + journaled = true + } + } +} +afterEach(() => jest.restoreAllMocks()) + +test('MySQL bootstrap uses current locks, bounded pages and idempotent reference edges before adoption', async () => { + const f = fixture() + try { + expect(await enabled(f.k)).toBe(false) + await install(f.k) + await install(f.k) + expect(f.progress()).toEqual({ id: 0, afterRowId: 257, complete: true }) + expect(f.edges).toHaveLength(427) + expect(f.edges.filter(edge => edge.tableId === 0)).toHaveLength(128) + expect(f.edges.filter(edge => edge.requestId === 0)).toHaveLength(171) + expect(f.edges.find(edge => edge.transactionId === 2 && edge.requestId !== 0 && edge.tableId === 1)).toEqual({ + transactionId: 2, + requestId: 4294967294, + tableId: 1, + rowId: 4294967295, + userId: 2 + }) + expect(f.triggers.size).toBe(14) + const ddl = f.queries.filter(q => q.sql.startsWith('CREATE TRIGGER')).map(q => q.sql) + expect(ddl[0]).toContain('snapshot_global_edge_delete') + expect(ddl[1]).toContain('snapshot_global_edge_insert') + expect(ddl.findIndex(sql => sql.includes('snapshot_global_tx_before_update'))).toBeLessThan( + ddl.findIndex(sql => sql.includes('snapshot_global_tx_insert')) + ) + expect(ddl.findIndex(sql => sql.includes('snapshot_global_req_before_update'))).toBeLessThan( + ddl.findIndex(sql => sql.includes('snapshot_global_req_insert')) + ) + expect(ddl.filter(sql => sql.includes('DECLARE requestedId INT UNSIGNED'))).toHaveLength(2) + expect(ddl.filter(sql => sql.includes('DECLARE requestedProof INT UNSIGNED'))).toHaveLength(2) + expect( + ddl.filter(sql => sql.includes('SELECT present FROM snapshot_global_guards WHERE proofId=NEW.rowId FOR SHARE')) + ).toHaveLength(1) + expect(ddl.filter(sql => sql.includes('ORDER BY transactionId FOR SHARE'))).toHaveLength(2) + expect(f.queries.filter(q => q.sql.startsWith('create table')).every(q => q.sql.includes('engine = InnoDB'))).toBe( + true + ) + expect(await enabled(f.k)).toBe(false) + f.setJournaled() + expect(await enabled(f.k)).toBe(true) + await remove(f.k) + await remove(f.k) + expect(f.tables).toEqual(new Set(['knex_migrations'])) + expect(f.triggers.size).toBe(0) + } finally { + await f.k.destroy() + } +}) + +test('MySQL trigger DDL preserves durable names, conditional updates and all current-read ownership bases', async () => { + const f = fixture() + try { + await install(f.k) + expect([...f.triggers].map(([name, row]) => [name, row.tableName, row.timing, row.event])).toEqual([ + ['snapshot_global_edge_delete', 'snapshot_global_edges', 'AFTER', 'DELETE'], + ['snapshot_global_edge_insert', 'snapshot_global_edges', 'AFTER', 'INSERT'], + ['snapshot_global_tx_delete', 'transactions', 'AFTER', 'DELETE'], + ['snapshot_global_tx_before_update', 'transactions', 'BEFORE', 'UPDATE'], + ['snapshot_global_req_delete', 'proven_tx_reqs', 'AFTER', 'DELETE'], + ['snapshot_global_req_before_update', 'proven_tx_reqs', 'BEFORE', 'UPDATE'], + ['snapshot_global_proof_delete', 'proven_txs', 'AFTER', 'DELETE'], + ['snapshot_global_proof_before_update', 'proven_txs', 'BEFORE', 'UPDATE'], + ['snapshot_global_proof_insert', 'proven_txs', 'AFTER', 'INSERT'], + ['snapshot_global_proof_after_update', 'proven_txs', 'AFTER', 'UPDATE'], + ['snapshot_global_tx_insert', 'transactions', 'AFTER', 'INSERT'], + ['snapshot_global_tx_after_update', 'transactions', 'AFTER', 'UPDATE'], + ['snapshot_global_req_insert', 'proven_tx_reqs', 'AFTER', 'INSERT'], + ['snapshot_global_req_after_update', 'proven_tx_reqs', 'AFTER', 'UPDATE'] + ]) + const body = (name: string): string => String(f.triggers.get('snapshot_global_' + name)?.body) + const transactionChange = + 'NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId)' + const requestChange = + 'NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId)' + for (const [name, condition] of [ + ['tx_before_update', transactionChange], + ['tx_after_update', transactionChange], + ['req_before_update', requestChange], + ['req_after_update', requestChange], + ['proof_before_update', 'NOT (OLD.provenTxId <=> NEW.provenTxId)'], + ['proof_after_update', 'NOT (OLD.provenTxId <=> NEW.provenTxId)'] + ]) { + expect(body(name)).toContain('IF ' + condition + ' THEN ') + expect(body(name)).toMatch(/ END IF; END$/) + } + for (const row of f.triggers.values()) expect(row.body).not.toContain('undefined') + for (const name of ['edge_delete', 'edge_insert', 'tx_delete', 'req_delete', 'proof_delete', 'proof_insert']) { + expect(body(name)).not.toContain(' END IF;') + } + expect(body('edge_insert')).toContain('ON DUPLICATE KEY UPDATE refs=snapshot_global_keys.refs+1;') + for (const name of ['tx_insert', 'tx_after_update']) { + const statement = body(name) + expect(statement).toContain('IF NEW.provenTxId IS NOT NULL THEN ') + expect(statement).toContain( + 'SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE;' + ) + expect(statement).toContain('IF requestedId IS NOT NULL THEN ') + expect(statement).toContain('IF requestedProof IS NOT NULL THEN ') + for (const tuple of [ + 'NEW.transactionId,0,1,NEW.provenTxId,NEW.userId', + 'NEW.transactionId,requestedId,0,requestedId,NEW.userId', + 'NEW.transactionId,requestedId,1,requestedProof,NEW.userId' + ]) + expect(statement).toContain( + 'VALUES (' + tuple + ') ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId;' + ) + for (const proof of ['NEW.provenTxId', 'requestedProof']) { + expect(statement).toContain( + 'INSERT INTO snapshot_global_guards (proofId,present) VALUES (' + + proof + + ',0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId;' + ) + expect(statement).toContain( + 'UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=' + + proof + + ';' + ) + } + } + for (const name of ['req_insert', 'req_after_update']) { + const statement = body(name) + expect(statement).toContain( + 'IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId;' + ) + expect(statement).toContain( + 'UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF;' + ) + expect( + statement.match(/ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId;/g) + ).toHaveLength(2) + } + for (const [name, presence] of [ + ['proof_delete', '0'], + ['proof_before_update', '0'], + ['proof_insert', '1'], + ['proof_after_update', '1'] + ]) + expect(body(name)).toContain('ON DUPLICATE KEY UPDATE present=' + presence + ';') + } finally { + await f.k.destroy() + } +}) + +test.each(['CASCADE', 'SET NULL', 'SET DEFAULT'])( + 'MySQL refuses implicit %s updates or deletes before DDL', + async rule => { + for (const field of ['updateRule', 'deleteRule']) { + const f = fixture((kind, _table, rows) => (kind === 'rules' ? [...rows, { ...rows[0], [field]: rule }] : rows)) + try { + await expect(install(f.k)).rejects.toThrow('requires explicit row mutations') + await expect(remove(f.k)).rejects.toThrow('requires explicit row mutations') + expect(f.queries.some(q => /^(create|alter|drop|insert|update|delete)/i.test(q.sql))).toBe(false) + } finally { + await f.k.destroy() + } + } + } +) + +test.each([ + ['engine', [], 'requires transactional tables'], + ['engine', [{ engine: 'MyISAM' }], 'requires transactional tables'], + ['engine', [{ engine: 'InnoDB' }, { engine: 'InnoDB' }], 'requires transactional tables'], + ['rules', null, 'requires explicit row mutations'], + ['sourceColumns', [], 'Unsupported snapshot global source column'], + ['indexes', null, 'Invalid snapshot global index metadata'], + ['indexes', [], 'Unsupported snapshot global source key'] +] as Array<[Kind, unknown, string]>)( + 'malformed source %s metadata refuses before migration', + async (kind, value, message) => { + const f = fixture((current, _table, rows) => (current === kind ? value : rows)) + try { + await expect(install(f.k)).rejects.toThrow(message) + await expect(remove(f.k)).rejects.toThrow(message) + } finally { + await f.k.destroy() + } + } +) + +test.each([{ type: 'bigint unsigned' }, { nullable: 'YES' }, { extra: 'VIRTUAL GENERATED' }, { name: 'other' }])( + 'MySQL source key metadata %j is refused', + async changed => { + const f = fixture((kind, _table, rows) => + kind === 'sourceColumns' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows + ) + try { + await expect(install(f.k)).rejects.toThrow('Unsupported snapshot global source column') + } finally { + await f.k.destroy() + } + } +) + +test.each([ + { charset: null }, + { collation: null }, + { type: 'varchar(63)' }, + { nullable: 'YES' }, + { extra: 'auto_increment' } +])('MySQL request text metadata %j is refused', async changed => { + const f = fixture((kind, table, rows) => + kind === 'sourceColumns' && table === 'proven_tx_reqs' + ? rows.map(row => (row.name === 'txid' ? { ...row, ...changed } : row)) + : rows + ) + try { + await expect(install(f.k)).rejects.toThrow('Unsupported snapshot global source') + } finally { + await f.k.destroy() + } +}) + +test.each([{ charset: 'latin1' }, { collation: 'utf8mb4_bin' }])( + 'MySQL incompatible source text %j is refused', + async changed => { + const f = fixture((kind, table, rows) => + kind === 'sourceColumns' && table === 'transactions' + ? rows.map(row => (row.name === 'txid' ? { ...row, ...changed } : row)) + : rows + ) + try { + await expect(install(f.k)).rejects.toThrow('require matching text definitions') + } finally { + await f.k.destroy() + } + } +) + +test.each([{ columnName: 'other' }, { direction: 'D' }, { prefix: 10 }, { nonUnique: 1 }])( + 'MySQL incomplete request lookup %j is refused', + async changed => { + const f = fixture((kind, table, rows) => + kind === 'indexes' && table === 'proven_tx_reqs' + ? rows.map(row => (row.name === 'source_txid' ? { ...row, ...changed } : row)) + : rows + ) + try { + await expect(install(f.k)).rejects.toThrow('requires complete transaction lookup indexes') + } finally { + await f.k.destroy() + } + } +) + +test.each([ + { name: 'other' }, + { type: 'bigint unsigned' }, + { nullable: 'YES' }, + { defaultValue: 0 }, + { extra: 'auto_increment' } +])('MySQL auxiliary column %j cannot be adopted', async changed => { + const f = fixture((kind, _table, rows) => + kind === 'columns' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows + ) + try { + await expect(install(f.k)).rejects.toThrow('table definition mismatch') + } finally { + await f.k.destroy() + } +}) + +test.each([{ columnName: 'other' }, { direction: 'D' }, { prefix: 10 }, { name: 'foreign_unique' }])( + 'MySQL auxiliary index %j cannot be adopted', + async changed => { + const f = fixture((kind, table, rows) => + kind === 'indexes' && table.startsWith('snapshot_') ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows + ) + try { + await expect(install(f.k)).rejects.toThrow('table definition mismatch') + } finally { + await f.k.destroy() + } + } +) + +test.each([{ event: 'UPDATE' }, { timing: 'BEFORE' }, { tableName: 'other' }, { body: 'BEGIN SELECT 1; END' }])( + 'MySQL altered trigger %j refuses removal before dropping objects', + async changed => { + let corrupt = false + const f = fixture((kind, _table, rows) => + corrupt && kind === 'triggers' && rows.length ? [{ ...rows[0], ...changed }] : rows + ) + try { + await install(f.k) + corrupt = true + await expect(install(f.k)).rejects.toThrow('trigger definition mismatch') + await expect(remove(f.k)).rejects.toThrow('trigger definition mismatch') + expect(f.triggers.size).toBe(14) + } finally { + await f.k.destroy() + } + } +) + +test('MySQL equivalent trigger whitespace resumes without replacing observers', async () => { + let reformatted = false + const f = fixture((kind, _table, rows) => + reformatted && kind === 'triggers' + ? rows.map(row => ({ ...row, body: '\n ' + String(row.body).replaceAll(' ', '\t \n') + '\n' })) + : rows + ) + try { + await install(f.k) + reformatted = true + await install(f.k) + expect(f.queries.filter(q => q.sql.startsWith('CREATE TRIGGER'))).toHaveLength(14) + await remove(f.k) + expect(f.triggers.size).toBe(0) + } finally { + await f.k.destroy() + } +}) + +test('MySQL refuses nested migration transactions before any database work', async () => { + const f = fixture() + try { + Object.defineProperty(f.k, 'isTransaction', { value: true }) + await expect(install(f.k)).rejects.toThrow('independent DDL') + await expect(remove(f.k)).rejects.toThrow('independent DDL') + expect(f.queries).toEqual([]) + } finally { + await f.k.destroy() + } +}) + +test.each([{ rows: [] }, { rows: [{ engine: 'MyISAM' }] }, { rows: [{ engine: 'InnoDB' }, { engine: 'InnoDB' }] }])( + 'MySQL auxiliary engine metadata %j refuses adoption', + async ({ rows: value }) => { + const f = fixture((kind, table, rows) => (kind === 'engine' && table.startsWith('snapshot_') ? value : rows)) + try { + await expect(install(f.k)).rejects.toThrow('table definition mismatch') + } finally { + await f.k.destroy() + } + } +) + +test('MySQL integer display widths preserve the supported source and auxiliary types', async () => { + const f = fixture((kind, _table, rows) => + kind === 'sourceColumns' || kind === 'columns' + ? rows.map(row => ({ ...row, type: String(row.type).replace(/^(bigint|int)\b/, '$1(10)') })) + : rows + ) + try { + await install(f.k) + f.setJournaled() + expect(await enabled(f.k)).toBe(true) + } finally { + await f.k.destroy() + } +}) + +test.each(['partial primary', 'extra unique', 'compound request lookup'])( + 'MySQL refuses %s index definitions even when their first column matches', + async altered => { + const f = fixture((kind, table, rows) => { + if (kind !== 'indexes') return rows + if (altered === 'partial primary' && table === 'snapshot_global_keys') + return rows.map((row, i) => (i === 1 ? { ...row, columnName: 'other' } : row)) + if (altered === 'extra unique' && table === 'snapshot_global_keys') + return [...rows, { ...rows[0], name: 'unexpected_unique', columnName: 'rowId' }] + if (altered === 'compound request lookup' && table === 'proven_tx_reqs') + return [...rows, { ...rows[1], columnName: 'provenTxReqId' }] + return rows + }) + try { + await expect(install(f.k)).rejects.toThrow( + altered === 'compound request lookup' + ? 'requires complete transaction lookup indexes' + : 'table definition mismatch' + ) + } finally { + await f.k.destroy() + } + } +) + +test.each( + [ + null, + [ + { event: 'DELETE', timing: 'AFTER', tableName: 'snapshot_global_edges', body: 'BEGIN END' }, + { event: 'DELETE', timing: 'AFTER', tableName: 'snapshot_global_edges', body: 'BEGIN END' } + ] + ].map(rows => ({ rows })) +)('MySQL malformed trigger metadata refuses %j', async ({ rows: value }) => { + const f = fixture((kind, _table, rows) => (kind === 'triggers' ? value : rows)) + try { + await expect(install(f.k)).rejects.toThrow('trigger') + } finally { + await f.k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.test.ts new file mode 100644 index 000000000..202568da9 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotGlobalIndexes.test.ts @@ -0,0 +1,452 @@ +import type { Knex } from 'knex' +import fc from 'fast-check' +import { runInSeries } from '../../utility/runInSeries' +import { + minimalGlobalDatabase, + expectGlobalMembership, + clearGlobalSource, + applyGlobalOperation +} from '../../../test/utils/snapshotGlobalFixtures' +import { + addSnapshotGlobalIndexes as installMigration, + removeSnapshotGlobalIndexes as remove, + readSnapshotGlobalIndexState as enabled, + SNAPSHOT_GLOBAL_INDEX_MIGRATION as migration +} from '../schema/snapshotGlobalIndexMigration' + +// A migration attempt must advance every committed source page. Fail a stalled +// cursor at the query boundary so its work settles before fixture cleanup. +async function install(k: Knex): Promise { + let previous = -1 + const advancing = (query: { sql: string; bindings?: readonly unknown[] }): void => { + if (!query.sql.includes('from `transactions` where `transactionId` > ? order by')) return + const cursor = Number(query.bindings?.at(-2)) + expect(cursor).toBeGreaterThan(previous) + previous = cursor + } + k.on('query', advancing) + try { + await installMigration(k) + } finally { + k.off('query', advancing) + } +} + +const databases: Knex[] = [] +async function fixture(collation = 'BINARY') { + const k = await minimalGlobalDatabase(collation) + databases.push(k) + return k +} +afterEach(async () => { + jest.restoreAllMocks() + await runInSeries(databases.splice(0), k => k.destroy()) +}) +async function journal(k: Knex, table = 'knex_migrations') { + await k.schema.createTable(table, t => { + void t.string('name') + }) + await k(table).insert({ name: migration }) +} + +test.each([0, 1, 255, 256, 257, 600])( + 'bootstrap/replay preserve %s source rows without double counting', + async count => { + const k = await fixture() + await k('proven_txs').insert([{ provenTxId: 7 }, { provenTxId: 9 }]) + await k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: 9 }) + await runInSeries( + Array.from({ length: Math.ceil(count / 200) }, (_, i) => i * 200), + async start => { + await k('transactions').insert( + Array.from({ length: Math.min(200, count - start) }, (_, i) => ({ + transactionId: start + i + 1, + userId: (i % 2) + 1, + txid: i % 2 ? 'a' : null, + provenTxId: i % 2 ? 7 : null + })) + ) + } + ) + const schema = () => + k('sqlite_master').whereIn('type', ['table', 'index']).select('name', 'type', 'sql').orderBy('name') + const before = await schema() + expect(await enabled(k)).toBe(false) + await install(k) + await expectGlobalMembership(k) + const keys = await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId']) + await install(k) + expect(await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId'])).toEqual(keys) + await k('snapshot_global_index_progress').where('id', 0).update({ afterRowId: 0, complete: false }) + await install(k) + await expectGlobalMembership(k) + expect(await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId'])).toEqual(keys) + expect(await k('snapshot_global_index_progress').first()).toEqual({ id: 0, afterRowId: count, complete: 1 }) + await remove(k) + await remove(k) + expect(await schema()).toEqual(before) + expect(await k('transactions')).toHaveLength(count) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) + } +) + +test.each(['BINARY', 'NOCASE', 'RTRIM'])( + 'generated reference counts and presence match the independent union under %s', + async collation => { + const k = await fixture(collation) + await install(k) + const operation = fc.record({ + kind: fc.integer({ min: 0, max: 8 }), + id: fc.integer({ min: 1, max: 5 }), + userId: fc.integer({ min: 1, max: 3 }), + otherId: fc.integer({ min: 1, max: 5 }), + nullable: fc.boolean() + }) + await fc.assert( + fc.asyncProperty(fc.array(operation, { minLength: 1, maxLength: 24 }), async schedule => { + await clearGlobalSource(k) + await expectGlobalMembership(k) + await runInSeries(schedule, async op => { + await applyGlobalOperation(k, op) + await expectGlobalMembership(k) + }) + }), + { numRuns: 300, seed: 3242026 } + ) + } +) + +test.each([ + ['transactions', 'transactionId', 1], + ['proven_tx_reqs', 'provenTxReqId', 5], + ['proven_txs', 'provenTxId', 7] +] as const)('unchanged membership in %s does not rewrite auxiliary rows', async (table, key, id) => { + const k = await fixture() + await k.schema.alterTable(table, schema => { + void schema.string('payload') + }) + await install(k) + expect(await k('sqlite_master').where('type', 'trigger').pluck('name')).toEqual( + expect.arrayContaining([ + 'snapshot_global_tx_after_update', + 'snapshot_global_req_after_update', + 'snapshot_global_proof_after_update' + ]) + ) + await k('proven_txs').insert({ provenTxId: 7 }) + await k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: 7 }) + await k('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: 7 }) + const changed = async (): Promise => Number((await k.raw('SELECT total_changes() AS count'))[0].count) + const before = await changed() + await k(table).where(key, id).update({ payload: 'updated payload' }) + expect((await changed()) - before).toBe(1) + const afterPayload = await changed() + await k(table) + .where(key, id) + .update({ [key]: id }) + expect((await changed()) - afterPayload).toBe(1) + await expectGlobalMembership(k) +}) + +test.each(['transactions', 'proven_tx_reqs'])( + 'replacing an identical %s row preserves existing reference edges without double counting', + async table => { + const k = await fixture() + await install(k) + await k('proven_txs').insert({ provenTxId: 7 }) + await k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: 7 }) + await k('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: 7 }) + const before = await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId']) + if (table === 'transactions') { + await k.raw('INSERT OR REPLACE INTO transactions(transactionId,userId,txid,provenTxId) VALUES (1,1,?,7)', ['a']) + } else { + await k.raw('INSERT OR REPLACE INTO proven_tx_reqs(provenTxReqId,txid,provenTxId) VALUES (5,?,7)', ['a']) + } + await expectGlobalMembership(k) + expect(await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId'])).toEqual(before) + } +) + +test('shared proofs remain owned until their final basis disappears and rolled-back changes leave no edges', async () => { + const k = await fixture() + await install(k) + await k('transactions').insert([ + { transactionId: 1, userId: 1, provenTxId: 7, txid: 'a' }, + { transactionId: 2, userId: 1, provenTxId: 7, txid: 'a' }, + { transactionId: 3, userId: 2, provenTxId: 7, txid: 'a' } + ]) + await k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: 7 }) + await expectGlobalMembership(k) + expect((await k('snapshot_global_keys').where({ tableId: 1, userId: 1, rowId: 7 }).first()).refs).toBe(4) + expect((await k('snapshot_global_keys').where({ tableId: 1, userId: 1, rowId: 7 }).first()).present).toBe(0) + await k('proven_txs').insert({ provenTxId: 7 }) + await expectGlobalMembership(k) + await expect( + k.transaction(async trx => { + await trx('transactions').delete() + throw new Error('synthetic rollback') + }) + ).rejects.toThrow('synthetic rollback') + await expectGlobalMembership(k) + await k('transactions').where('transactionId', 1).delete() + await expectGlobalMembership(k) + expect((await k('snapshot_global_keys').where({ tableId: 1, userId: 1, rowId: 7 }).first()).refs).toBe(2) + await k('transactions').where('transactionId', 2).delete() + await expectGlobalMembership(k) + expect(await k('snapshot_global_keys').where({ tableId: 1, userId: 1 })).toEqual([]) + await k('proven_txs').where('provenTxId', 7).delete() + await expectGlobalMembership(k) + await k('proven_txs').insert({ provenTxId: 7 }) + await expectGlobalMembership(k) + await k('transactions').delete() + await expectGlobalMembership(k) + expect(await k('snapshot_global_keys')).toEqual([]) + expect(await k('snapshot_global_guards')).toEqual([]) +}) + +test.each(['BINARY', 'NOCASE', 'RTRIM'])( + 'full-width IDs and collation-equivalent text changes preserve references under %s', + async collation => { + const k = await fixture(collation) + await install(k) + await k('proven_txs').insert({ provenTxId: 4294967295 }) + await k('proven_tx_reqs').insert({ provenTxReqId: 4294967294, txid: 'A', provenTxId: 4294967295 }) + await k('transactions').insert({ transactionId: 4294967293, userId: 4294967292, txid: 'A', provenTxId: 4294967295 }) + await expectGlobalMembership(k) + await runInSeries(['a', 'a ', 'é', 'e\u0301', 'A\0B', '😀'.repeat(64)], async txid => { + await k('proven_tx_reqs').where('provenTxReqId', 4294967294).update({ txid }) + await expectGlobalMembership(k) + await k('transactions').where('transactionId', 4294967293).update({ txid }) + await expectGlobalMembership(k) + }) + await k('snapshot_global_index_progress').where('id', 0).update({ afterRowId: 0, complete: false }) + await install(k) + await expectGlobalMembership(k) + } +) + +test.each([0, -1])( + 'bootstrap refuses an initial unsupported transaction ID %s and can resume after repair', + async id => { + const k = await fixture() + await k('transactions').insert({ transactionId: id, userId: 1, txid: null, provenTxId: null }) + await expect(install(k)).rejects.toThrow('Invalid snapshot global source key') + expect((await k('snapshot_global_index_progress').first()).complete).toBe(0) + await k('transactions').where('transactionId', id).delete() + await install(k) + await expectGlobalMembership(k) + } +) + +test.each(['userId', 'provenTxId'])('bootstrap rejects unsupported %s without publishing progress', async field => { + const k = await fixture() + await k('transactions').insert({ transactionId: 1, userId: 1, txid: null, provenTxId: null, [field]: 0 }) + await expect(install(k)).rejects.toThrow('Invalid snapshot global source key') + expect((await k('snapshot_global_index_progress').first()).afterRowId).toBe(0) +}) + +test('oversized SQLite transaction text is rejected before loading it as a lookup key', async () => { + const k = await fixture() + await k('transactions').insert({ transactionId: 1, userId: 1, txid: 'a'.repeat(257), provenTxId: null }) + await expect(install(k)).rejects.toThrow('Invalid snapshot global transaction key') + expect(await k('snapshot_global_edges')).toEqual([]) + expect((await k('snapshot_global_index_progress').first()).afterRowId).toBe(0) +}) + +test.each(['knex_migrations', 'custom_migrations'])( + 'adoption requires exact owned tables and completed journal %s', + async tableName => { + const k = await fixture() + await install(k) + expect(await enabled(k, { tableName })).toBe(false) + await journal(k, tableName) + expect(await enabled(k, { tableName })).toBe(true) + await k('snapshot_global_index_progress').where('id', 0).update({ complete: false }) + await expect(enabled(k, { tableName })).rejects.toThrow('incomplete') + await install(k) + expect(await enabled(k, { tableName })).toBe(true) + await remove(k) + await expect(enabled(k, { tableName })).rejects.toThrow('incomplete') + } +) + +test('altered trigger definitions are refused by install and removal before source changes', async () => { + const k = await fixture() + await install(k) + await k.raw('DROP TRIGGER snapshot_global_tx_insert') + await k.raw('CREATE TRIGGER snapshot_global_tx_insert AFTER INSERT ON transactions BEGIN SELECT 1; END') + await expect(install(k)).rejects.toThrow('trigger definition mismatch') + await expect(remove(k)).rejects.toThrow('trigger definition mismatch') + expect(await k.schema.hasTable('snapshot_global_keys')).toBe(true) +}) + +test.each([ + 'snapshot_global_guards', + 'snapshot_global_keys', + 'snapshot_global_edges', + 'snapshot_global_index_progress' +])('an unexpected %s column is refused for resume, adoption and removal', async table => { + const k = await fixture() + await install(k) + await journal(k) + await k.schema.alterTable(table, t => { + void t.integer('unexpected') + }) + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(enabled(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') + expect(await k.schema.hasTable('snapshot_global_keys')).toBe(true) +}) + +test.each([ + 'transactionId INTEGER NOT NULL, userId INTEGER NOT NULL, provenTxId INTEGER, txid VARCHAR(64)', + 'transactionId INTEGER NOT NULL, userId INTEGER NOT NULL, provenTxId INTEGER, txid VARCHAR(64), PRIMARY KEY(transactionId,userId)', + 'otherId INTEGER NOT NULL PRIMARY KEY, transactionId INTEGER, userId INTEGER NOT NULL, provenTxId INTEGER, txid VARCHAR(64)' +])('unsupported SQLite source key refuses before auxiliary DDL: %s', async columns => { + const k = await fixture() + await k.schema.dropTable('transactions') + await k.raw(`CREATE TABLE transactions(${columns})`) + await expect(install(k)).rejects.toThrow('Unsupported snapshot global source key') + expect(await k.schema.hasTable('snapshot_global_guards')).toBe(false) +}) + +test.each([ + 'transactionId INTEGER PRIMARY KEY, userId TEXT NOT NULL, provenTxId INTEGER, txid VARCHAR(64)', + 'transactionId INTEGER PRIMARY KEY, userId INTEGER, provenTxId INTEGER, txid VARCHAR(64)', + 'transactionId INTEGER PRIMARY KEY, userId INTEGER NOT NULL, txid VARCHAR(64)', + 'transactionId INTEGER PRIMARY KEY, userId INTEGER NOT NULL, provenTxId INTEGER, txid TEXT', + 'transactionId INTEGER PRIMARY KEY, userId INTEGER NOT NULL, provenTxId INTEGER, txid VARCHAR(64) NOT NULL', + 'transactionId INTEGER PRIMARY KEY, userId INTEGER NOT NULL, provenTxId INTEGER GENERATED ALWAYS AS (userId) VIRTUAL, txid VARCHAR(64)' +])('unsupported SQLite source column refuses before auxiliary DDL: %s', async columns => { + const k = await fixture() + await k.schema.dropTable('transactions') + await k.raw(`CREATE TABLE transactions(${columns})`) + await expect(install(k)).rejects.toThrow('Unsupported snapshot global source column') + expect(await k.schema.hasTable('snapshot_global_guards')).toBe(false) +}) + +test.each(['(userId)', '(txid DESC)', '(txid COLLATE NOCASE)', '(txid) WHERE userId = 1'])( + 'an incomplete or differently ordered source lookup is refused: %s', + async definition => { + const k = await fixture() + await k.schema.dropTable('transactions') + await k.raw( + 'CREATE TABLE transactions(transactionId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,provenTxId INTEGER,txid VARCHAR(64))' + ) + await k.raw(`CREATE INDEX synthetic_txid_lookup ON transactions${definition}`) + await expect(install(k)).rejects.toThrow('requires complete transaction lookup indexes') + expect(await k.schema.hasTable('snapshot_global_guards')).toBe(false) + } +) + +test('SQLite source txid comparison metadata must agree between transactions and requests', async () => { + const k = await fixture() + await k.schema.dropTable('transactions') + await k.raw( + 'CREATE TABLE transactions(transactionId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,provenTxId INTEGER,txid VARCHAR(64) COLLATE NOCASE)' + ) + await k.raw('CREATE INDEX synthetic_txid_lookup ON transactions(txid)') + await expect(install(k)).rejects.toThrow('require matching text definitions') + expect(await k.schema.hasTable('snapshot_global_guards')).toBe(false) +}) + +test.each([ + '(tableId,userId,present,rowId DESC)', + '(tableId,userId,present,rowId COLLATE NOCASE)', + '(tableId,userId,present)', + '(tableId,present,userId,rowId)', + '(tableId,userId,present,rowId) WHERE refs > 0' +])('altered auxiliary page order refuses resume, adoption and down: %s', async definition => { + const k = await fixture() + await install(k) + await journal(k) + await k.raw('DROP INDEX snapshot_global_page') + await k.raw(`CREATE INDEX snapshot_global_page ON snapshot_global_keys${definition}`) + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(enabled(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') + expect(await k.schema.hasTable('snapshot_global_keys')).toBe(true) +}) + +test.each([{ afterRowId: -1 }, { afterRowId: 1.5 }, { complete: 2 }, { id: 1 }])( + 'malformed bootstrap progress %j refuses resume and journal adoption', + async changed => { + const k = await fixture() + await install(k) + await journal(k) + await k('snapshot_global_index_progress').where('id', 0).update(changed) + await expect(enabled(k)).rejects.toThrow('incomplete') + if (!('id' in changed)) await expect(install(k)).rejects.toThrow('Invalid snapshot global bootstrap position') + } +) + +test('qualified migration journals and empty progress refuse or adopt the same schema', async () => { + const k = await fixture() + expect(await enabled(k, { schemaName: 'main' })).toBe(false) + await install(k) + await journal(k) + expect(await enabled(k, { schemaName: 'main' })).toBe(true) + await k('snapshot_global_index_progress').delete() + await expect(enabled(k, { schemaName: 'main' })).rejects.toThrow('incomplete') +}) + +test('bootstrap preserves missing request and missing proof distinctions', async () => { + const k = await fixture() + await k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: null }) + await k('transactions').insert([ + { transactionId: 1, userId: 1, txid: 'missing', provenTxId: null }, + { transactionId: 2, userId: 1, txid: 'missing', provenTxId: 7 }, + { transactionId: 3, userId: 1, txid: 'a', provenTxId: null } + ]) + await install(k) + await expectGlobalMembership(k) + expect(await k('snapshot_global_keys').orderBy('tableId')).toEqual([ + { tableId: 0, userId: 1, rowId: 5, refs: 1, present: 1 }, + { tableId: 1, userId: 1, rowId: 7, refs: 1, present: 0 } + ]) +}) + +test('unexpected unique auxiliary constraints refuse adoption before they can reject valid source writes', async () => { + const k = await fixture() + await install(k) + await journal(k) + await k.raw('CREATE UNIQUE INDEX synthetic_extra_unique ON snapshot_global_keys(rowId)') + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(enabled(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') +}) + +test('inconsistent SQLite primary-index metadata refuses adoption instead of assuming its key exists', async () => { + const k = await fixture() + await install(k) + const raw = k.client.raw.bind(k.client) + jest.spyOn(k.client, 'raw').mockImplementation((...args) => { + const query = raw(...args) + if (args[0] === 'PRAGMA index_list(??)' && Array.isArray(args[1]) && args[1][0] === 'snapshot_global_keys') { + return query.then((rows: Array<{ origin: string }>) => + rows.filter(row => row.origin !== 'pk') + ) as unknown as Knex.Raw + } + return query + }) + await expect(install(k)).rejects.toThrow('table definition mismatch') +}) + +test.each([ + 'other integer NOT NULL PRIMARY KEY, present boolean NOT NULL', + 'proofId bigint NOT NULL PRIMARY KEY, present boolean NOT NULL', + 'proofId integer PRIMARY KEY, present boolean NOT NULL', + 'proofId integer NOT NULL PRIMARY KEY DEFAULT 0, present boolean NOT NULL', + 'proofId integer NOT NULL, present boolean NOT NULL PRIMARY KEY', + 'proofId integer NOT NULL PRIMARY KEY, present boolean NOT NULL GENERATED ALWAYS AS (1) STORED' +])('same-size altered auxiliary columns refuse resume, adoption and removal: %s', async definition => { + const k = await fixture() + await install(k) + await journal(k) + await k.schema.dropTable('snapshot_global_guards') + await k.raw(`CREATE TABLE snapshot_global_guards(${definition})`) + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(enabled(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') + expect(await k.schema.hasTable('snapshot_global_keys')).toBe(true) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.integration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.integration.test.ts new file mode 100644 index 000000000..63d5d7ffa --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.integration.test.ts @@ -0,0 +1,217 @@ +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { runInSeries } from '../../utility/runInSeries' +import { seedArchiveClosure } from '../../../test/utils/snapshotArchiveFixtures' +import { snapshotArchiveTables } from './archive/SnapshotArchive' +import { openKnexSnapshotArchiveSource, type SnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import { + addSnapshotProfileIndexes, + removeSnapshotProfileIndexes, + SNAPSHOT_PROFILE_INDEX_MIGRATION +} from '../schema/snapshotProfileIndexMigration' + +const identity = '02' + '11'.repeat(32) +const stores: StorageKnex[] = [] +const writers: Knex[] = [] +const directories: string[] = [] +const dates = { created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-01T00:00:00.000Z' } +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-profile-index-')) + directories.push(directory) + const options = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + } + const k = knex(options) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + stores.push(source) + await k.raw('PRAGMA journal_mode=WAL') + await migrateBeforeSqliteGeneration(source, 'profile index fixture', 'synthetic-profile-index') + await source.makeAvailable() + // Start with the immediately preceding schema on both old and new source. + await removeSnapshotProfileIndexes(k) + await k('knex_migrations').where('name', SNAPSHOT_PROFILE_INDEX_MIGRATION).delete() + const { user } = await source.findOrInsertUser(identity) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, other.userId) + const writer = knex(options) + writers.push(writer) + return { source, writer, k, userId: user.userId, otherId: other.userId } +} +async function journal(k: Knex): Promise { + await k('knex_migrations').insert({ name: SNAPSHOT_PROFILE_INDEX_MIGRATION, batch: 99, migration_time: new Date() }) +} +async function pages(view: WalletReadSnapshot | SnapshotArchiveSource) { + expect(Object.hasOwn(view, 'profileIndexes')).toBe(false) + if ('validateClosure' in view) await view.validateClosure() + const result: Record = {} + await runInSeries(snapshotArchiveTables, async table => { + let cursor: WalletSnapshotCursor | undefined + const selected: unknown[] = [] + let complete = false + for (let pageNumber = 0; pageNumber < 25 && !complete; pageNumber++) { + const page = await view.readPage(table, cursor, { maxRows: 1, maxBytes: 131072 }) + selected.push({ rows: page.rows, after: page.cursor?.after, payloadBytes: page.payloadBytes, done: page.done }) + complete = page.done + cursor = page.cursor + } + expect(complete).toBe(true) + result[table] = selected + }) + return result +} +afterEach(async () => { + await runInSeries(writers.splice(0), k => k.destroy()) + await runInSeries(stores.splice(0), source => source.destroy()) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('indexed ordinary/archive pages preserve all13tables and original source indexes/legacy offsets', async () => { + const { source, k, userId } = await fixture() + const legacy = async () => { + const all: Record = {} + await runInSeries( + [ + ['findTransactions', 'transactionId'], + ['findOutputs', 'outputId'], + ['findCertificates', 'certificateId'], + ['findTxLabels', 'txLabelId'], + ['findOutputBaskets', 'basketId'], + ['findOutputTags', 'outputTagId'], + ['findCommissions', 'commissionId'], + ['findSyncStates', 'syncStateId'] + ] as const, + async ([method, key]) => { + all[method] = [] + await runInSeries([0, 1], async offset => { + const rows = await source[method]({ partial: { userId }, paged: { limit: 1, offset } }) + all[method].push(rows.map(row => (row as unknown as Record)[key])) + }) + } + ) + return all + } + const standard = async () => + await k('sqlite_master') + .whereIn('type', ['table', 'index']) + .whereNotIn('tbl_name', ['snapshot_profile_keys', 'snapshot_profile_index_progress']) + .select('type', 'name', 'tbl_name', 'sql') + .orderBy('name') + const originalIndexes = await standard() + const originalOffsets = await legacy() + const old = await source.openWalletReadSnapshot(identity) + let baseline + try { + baseline = await pages(old) + } finally { + await old.close() + } + await addSnapshotProfileIndexes(k) + expect(await standard()).toEqual(originalIndexes) + expect(await legacy()).toEqual(originalOffsets) + const queries: Array<{ sql: string; bindings: Knex.RawBinding[] }> = [] + const listen = (query: { sql: string; bindings: Knex.RawBinding[] }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_profile_keys') + ) + queries.push(query) + } + k.on('query', listen) + try { + const partial = await source.openWalletReadSnapshot(identity) + try { + expect(await pages(partial)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await partial.close() + } + await journal(k) + await runInSeries( + [() => source.openWalletReadSnapshot(identity), () => openKnexSnapshotArchiveSource(source, identity)], + async open => { + const view = await open() + try { + expect(await pages(view)).toEqual(baseline) + } finally { + await view.close() + } + } + ) + expect(queries.length).toBeGreaterThan(0) + } finally { + k.off('query', listen) + } + // Explain only page queries; closure may legitimately visit its relational parents. + await runInSeries( + queries.filter(query => query.sql.includes('__snapshotBytes') || query.sql.includes('.*')), + async query => { + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + query.sql, query.bindings) + expect(plan.some(row => row.detail.includes('SEARCH snapshot_profile_keys USING COVERING INDEX'))).toBe(true) + expect(plan.some(row => /SCAN |TEMP B-TREE/.test(row.detail))).toBe(false) + } + ) +}) + +test.each(['ordinary', 'archive'] as const)( + '%s mode and all13table pages stay bound across independent journal/profile/progress writes', + async kind => { + const { source, k, writer, userId, otherId } = await fixture() + await addSnapshotProfileIndexes(k) + const open = async () => + kind === 'ordinary' + ? await source.openWalletReadSnapshot(identity) + : await openKnexSnapshotArchiveSource(source, identity) + const queries: string[] = [] + const listen = (query: { sql: string }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_profile_keys') + ) + queries.push(query.sql) + } + k.on('query', listen) + const old = await open() + let baseline + try { + baseline = await pages(old) + queries.length = 0 + await journal(writer) + expect(await pages(old)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await old.close() + } + const indexed = await open() + try { + await writer.transaction(async trx => { + await trx('snapshot_profile_index_progress').where('snapshotTableId', 3).update({ complete: 0 }) + await trx('tx_labels').where('txLabelId', 3).update({ userId: otherId, label: 'moved' }) + await trx('tx_labels').insert({ ...dates, txLabelId: 4, userId, label: 'new after view', isDeleted: false }) + }) + expect(await pages(indexed)).toEqual(baseline) + expect(queries.length).toBeGreaterThan(0) + } finally { + await indexed.close() + k.off('query', listen) + } + await expect(open()).rejects.toThrow('migration is incomplete') + await writer('snapshot_profile_index_progress').where('snapshotTableId', 3).update({ complete: 1 }) + const fresh = await open() + try { + expect((await fresh.readPage('txLabels')).rows.map(row => row.txLabelId)).toEqual([1, 4]) + } finally { + await fresh.close() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.migration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.migration.test.ts new file mode 100644 index 000000000..b920fe426 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.migration.test.ts @@ -0,0 +1,87 @@ +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { KnexMigrations } from '../schema/KnexMigrations' +import { SNAPSHOT_GLOBAL_INDEX_MIGRATION } from '../schema/snapshotGlobalIndexMigration' +import { + readSnapshotProfileIndexState, + SNAPSHOT_PROFILE_INDEX_MIGRATION +} from '../schema/snapshotProfileIndexMigration' + +test('registered profile bootstrap survives reopening and publishes its journal only after all pages commit', async () => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-profile-migration-')) + const options = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + } + let database = knex(options) + let source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: database }) + const migrationSource = new KnexMigrations('test', 'profile migration', 'synthetic-profile-migration', 1024) + try { + expect((await migrationSource.getMigration(SNAPSHOT_PROFILE_INDEX_MIGRATION)).config).toEqual({ + transaction: false + }) + await migrateBeforeSqliteGeneration(source, 'profile migration', 'synthetic-profile-migration') + await source.makeAvailable() + await database.migrate.down({ migrationSource, name: SNAPSHOT_PROFILE_INDEX_MIGRATION, disableTransactions: false }) + const priorJournal = await database('knex_migrations').orderBy('id') + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + for (let batch = 0; batch < 3; batch++) { + await database('tx_labels').insert( + Array.from({ length: 200 }, (_, index) => ({ + userId: user.userId, + label: `label-${batch * 200 + index}`, + isDeleted: false, + created_at: new Date('2026-01-01'), + updated_at: new Date('2026-01-01') + })) + ) + } + let pages = 0 + const failure = new Error('synthetic interrupted profile bootstrap') + const interrupt = (query: { sql: string }): void => { + if (query.sql.startsWith('select `txLabelId`, `userId` from `tx_labels`') && ++pages === 2) throw failure + } + database.on('query', interrupt) + try { + await expect(migrateBeforeSqliteGeneration(source, 'profile migration', 'synthetic-profile-migration')).rejects.toBe(failure) + } finally { + database.off('query', interrupt) + } + expect(await database('knex_migrations').orderBy('id')).toEqual(priorJournal) + expect(await database('snapshot_profile_index_progress').where('snapshotTableId', 3).first()).toEqual({ + snapshotTableId: 3, + afterRowId: 256, + complete: 0 + }) + expect(await readSnapshotProfileIndexState(database)).toBe(false) + expect((await database.raw('PRAGMA foreign_keys'))[0].foreign_keys).toBe(1) + await source.destroy() + database = knex(options) + source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: database }) + await expect(migrateBeforeSqliteGeneration(source, 'profile migration', 'synthetic-profile-migration')).resolves.toBe( + SNAPSHOT_GLOBAL_INDEX_MIGRATION + ) + expect(await readSnapshotProfileIndexState(database)).toBe(true) + expect(await database('snapshot_profile_keys').where('snapshotTableId', 3).count({ count: '*' }).first()).toEqual({ + count: 600 + }) + expect( + await database('knex_migrations').where('name', SNAPSHOT_PROFILE_INDEX_MIGRATION).count({ count: '*' }).first() + ).toEqual({ count: 1 }) + await database.migrate.down({ migrationSource, name: SNAPSHOT_PROFILE_INDEX_MIGRATION, disableTransactions: false }) + expect(await database.schema.hasTable('snapshot_profile_keys')).toBe(false) + expect(await database.schema.hasTable('snapshot_profile_index_progress')).toBe(false) + expect(await database('knex_migrations').orderBy('id')).toEqual(priorJournal) + expect(await database('tx_labels').count({ count: '*' }).first()).toEqual({ count: 600 }) + } finally { + await source.destroy() + await rm(directory, { recursive: true, force: true }) + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.mysql.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.mysql.test.ts new file mode 100644 index 000000000..c466ce9f5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.mysql.test.ts @@ -0,0 +1,315 @@ +import { knex } from 'knex' +import { + addSnapshotProfileIndexes, + removeSnapshotProfileIndexes, + readSnapshotProfileIndexState, + SNAPSHOT_PROFILE_INDEX_MIGRATION, + snapshotProfileTables +} from '../schema/snapshotProfileIndexMigration' + +type MetadataKind = 'columns' | 'indexes' | 'triggers' +type Metadata = Array> + +/** Installed MySQL compiler and transaction protocol; native fixtures separately prove database semantics. */ +function mysqlFixture(change: (kind: MetadataKind, rows: Metadata) => unknown = (_kind, rows) => rows) { + const k = knex({ client: 'mysql2' }) + const queries: Array<{ sql: string; values: unknown[] }> = [] + const tables = new Set(['knex_migrations']) + const triggers = new Map>() + const progress = new Map() + const keys: Array<{ snapshotTableId: number; snapshotUserId: number; snapshotRowId: number }> = [] + let journaled = false + const answer = (sql: string, values: unknown[]): unknown => { + queries.push({ sql, values }) + if (sql.startsWith('select * from information_schema.tables')) + return tables.has(String(values[0])) ? [{ TABLE_NAME: values[0] }] : [] + if (sql.startsWith('create table')) { + tables.add(sql.match(/^create table `([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('drop table')) { + tables.delete(sql.match(/`([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('SELECT COLUMN_NAME')) { + const isKeys = values[0] === 'snapshot_profile_keys' + const names = isKeys + ? ['snapshotTableId', 'snapshotUserId', 'snapshotRowId'] + : ['snapshotTableId', 'afterRowId', 'complete'] + const types = isKeys ? ['int', 'int unsigned', 'int unsigned'] : ['int', 'int unsigned', 'tinyint'] + return change( + 'columns', + names.map((name, i) => ({ name, type: types[i], nullable: 'NO', defaultValue: null, extra: '' })) + ) + } + if (sql.startsWith('SELECT INDEX_NAME')) { + return change( + 'indexes', + (values[0] === 'snapshot_profile_keys' + ? ['snapshotTableId', 'snapshotUserId', 'snapshotRowId'] + : ['snapshotTableId'] + ).map(columnName => ({ name: 'PRIMARY', columnName, nonUnique: 0, direction: 'A', prefix: null })) + ) + } + if (sql.startsWith('SELECT EVENT_MANIPULATION')) + return change('triggers', triggers.has(String(values[0])) ? [triggers.get(String(values[0]))!] : []) + if (sql.startsWith('CREATE TRIGGER')) { + const [, name, event, tableName, body] = sql.match( + /^CREATE TRIGGER (\w+) AFTER (\w+) ON (\w+) FOR EACH ROW (.*)$/ + )! + triggers.set(name, { event, timing: 'AFTER', tableName, body }) + return [] + } + if (sql.startsWith('DROP TRIGGER')) { + triggers.delete(sql.split(' ').at(-1)!) + return [] + } + if (sql.startsWith('insert ignore into `snapshot_profile_index_progress`')) { + const [afterRowId, complete, snapshotTableId] = values as [number, boolean, number] + if (!progress.has(snapshotTableId)) progress.set(snapshotTableId, { snapshotTableId, afterRowId, complete }) + return { affectedRows: 1, insertId: 0 } + } + if (sql.startsWith('select * from `snapshot_profile_index_progress`')) { + expect(sql).toMatch(/for update$/) + const row = progress.get(Number(values[0])) + return row === undefined ? [] : [{ ...row }] + } + if (sql.startsWith('select `snapshotTableId`, `afterRowId`, `complete`')) + return [...progress.values()].map(row => ({ ...row })).slice(0, Number(values[0])) + if (sql.startsWith('update `snapshot_profile_index_progress`')) { + const [afterRowId, complete, tableId] = values as [number, boolean, number] + Object.assign(progress.get(tableId)!, { afterRowId, complete }) + return { affectedRows: 1 } + } + if (sql.startsWith('insert ignore into `snapshot_profile_keys`')) { + for (let i = 0; i < values.length; i += 3) { + const [snapshotRowId, snapshotTableId, snapshotUserId] = values.slice(i, i + 3) as number[] + keys.push({ snapshotTableId, snapshotUserId, snapshotRowId }) + } + return { affectedRows: values.length / 3, insertId: 0 } + } + if (sql.startsWith('select `name` from `knex_migrations`')) + return journaled ? [{ name: SNAPSHOT_PROFILE_INDEX_MIGRATION }] : [] + if (/^(BEGIN|COMMIT|ROLLBACK)/.test(sql)) return [] + const source = snapshotProfileTables.find(({ table }) => sql.includes('from `' + table + '`')) + if (source !== undefined) { + expect(sql).toMatch(/for update$/) + expect(values[1]).toBe(256) + return [ + { [source.key]: 1, userId: 1 }, + { [source.key]: 3, userId: 2 } + ].filter(row => row[source.key] > Number(values[0])) + } + throw new Error('Unexpected synthetic driver query: ' + sql) + } + const connection = { + query( + query: { sql: string }, + values: unknown[], + callback: (error: Error | null, rows?: unknown, fields?: unknown[]) => void + ) { + try { + callback(null, answer(query.sql, values), []) + } catch (error) { + callback(error as Error) + } + } + } + jest.spyOn(k.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(k.client, 'releaseConnection').mockResolvedValue(undefined) + return { + k, + queries, + keys, + progress, + triggers, + tables, + journal: () => { + journaled = true + } + } +} + +afterEach(() => jest.restoreAllMocks()) + +test('MySQL DDL and locked bounded bootstrap preserve completion ordering and support repeat/removal', async () => { + const f = mysqlFixture() + try { + await addSnapshotProfileIndexes(f.k) + expect(f.triggers.size).toBe(24) + expect(f.keys).toEqual( + snapshotProfileTables.flatMap((_table, snapshotTableId) => [ + { snapshotTableId, snapshotUserId: 1, snapshotRowId: 1 }, + { snapshotTableId, snapshotUserId: 2, snapshotRowId: 3 } + ]) + ) + expect([...f.progress.values()]).toEqual( + snapshotProfileTables.map((_table, snapshotTableId) => ({ snapshotTableId, afterRowId: 3, complete: true })) + ) + expect( + f.queries.filter(query => query.sql.includes('for update') && !query.sql.includes('snapshot_profile')) + ).toHaveLength(8) + expect(f.queries.find(query => query.sql.startsWith('create table `snapshot_profile_keys`'))!.sql).toContain( + 'primary key (`snapshotTableId`, `snapshotUserId`, `snapshotRowId`)' + ) + expect(f.queries.filter(query => query.sql === 'BEGIN;')).toHaveLength(8) + expect(f.queries.filter(query => query.sql === 'COMMIT;')).toHaveLength(8) + f.journal() + expect(await readSnapshotProfileIndexState(f.k)).toBe(true) + await addSnapshotProfileIndexes(f.k) + expect(f.keys).toHaveLength(16) + await removeSnapshotProfileIndexes(f.k) + expect(f.triggers.size).toBe(0) + expect(f.tables.has('snapshot_profile_keys')).toBe(false) + } finally { + await f.k.destroy() + } +}) + +test.each([ + ['columns', []], + ['columns', null], + ['indexes', []], + ['indexes', null] +] as const)('MySQL missing/non-array %s metadata refuses before installing triggers', async (kind, replacement) => { + const f = mysqlFixture((current, rows) => (current === kind ? replacement : rows)) + try { + await expect(addSnapshotProfileIndexes(f.k)).rejects.toThrow('table definition mismatch') + expect(f.triggers.size).toBe(0) + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } +}) + +test.each([ + { name: 'unexpected' }, + { type: 'bigint' }, + { nullable: 'YES' }, + { defaultValue: 0 }, + { extra: 'auto_increment' } +])('MySQL column metadata mismatch %j refuses without bootstrap', async changed => { + const f = mysqlFixture((kind, rows) => (kind === 'columns' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows)) + try { + await expect(addSnapshotProfileIndexes(f.k)).rejects.toThrow('table definition mismatch') + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } +}) + +test.each([{ direction: 'D' }, { prefix: 1 }, { columnName: 'unexpected' }])( + 'MySQL primary-key mismatch %j refuses adoption', + async changed => { + const f = mysqlFixture((kind, rows) => (kind === 'indexes' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows)) + try { + await expect(addSnapshotProfileIndexes(f.k)).rejects.toThrow('table definition mismatch') + } finally { + await f.k.destroy() + } + } +) + +test('MySQL extra unique constraints refuse while nonunique indexes remain compatible', async () => { + for (const nonUnique of [0, 1]) { + const f = mysqlFixture((kind, rows) => + kind === 'indexes' ? [...rows, { name: 'extra', columnName: 'snapshotRowId', nonUnique }] : rows + ) + try { + if (nonUnique === 0) await expect(addSnapshotProfileIndexes(f.k)).rejects.toThrow('table definition mismatch') + else { + await addSnapshotProfileIndexes(f.k) + expect(f.keys).toHaveLength(16) + } + } finally { + await f.k.destroy() + } + } +}) + +test('a MySQL migration in an outer transaction refuses before DDL can commit it implicitly', async () => { + const f = mysqlFixture() + try { + await f.k.transaction(async trx => { + await expect(addSnapshotProfileIndexes(trx)).rejects.toThrow('independent DDL') + await expect(removeSnapshotProfileIndexes(trx)).rejects.toThrow('independent DDL') + }) + expect(f.queries.every(query => /^(BEGIN|COMMIT)/.test(query.sql))).toBe(true) + } finally { + await f.k.destroy() + } +}) + +test.each([ + { event: 'UNKNOWN' }, + { timing: 'BEFORE' }, + { tableName: 'foreign_table' }, + { body: 'BEGIN SELECT 1; END' } +])('MySQL stored trigger mismatch %j refuses adoption/removal without deleting definitions', async changed => { + let corrupt = false + const f = mysqlFixture((kind, rows) => (corrupt && kind === 'triggers' ? [{ ...rows[0], ...changed }] : rows)) + try { + await addSnapshotProfileIndexes(f.k) + corrupt = true + await expect(addSnapshotProfileIndexes(f.k)).rejects.toThrow('trigger definition mismatch') + await expect(removeSnapshotProfileIndexes(f.k)).rejects.toThrow('trigger definition mismatch') + expect(f.triggers.size).toBe(24) + expect(f.tables.has('snapshot_profile_keys')).toBe(true) + } finally { + await f.k.destroy() + } +}) + +test.each([null, [undefined], [{}, {}]])( + 'MySQL malformed trigger metadata %p refuses before creation', + async replacement => { + const f = mysqlFixture((kind, rows) => (kind === 'triggers' ? replacement : rows)) + try { + await expect(addSnapshotProfileIndexes(f.k)).rejects.toThrow(/trigger (metadata|definition mismatch)/) + expect(f.triggers.size).toBe(0) + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } + } +) + +test('MySQL native-width metadata and harmless body whitespace do not require recreation', async () => { + const f = mysqlFixture((kind, rows) => + kind === 'columns' + ? rows.map(row => ({ ...row, type: String(row.type).replace('int', 'int(11)') })) + : kind === 'triggers' + ? rows.map(row => ({ ...row, body: String(row.body).replace(/ /g, ' \n ') })) + : rows + ) + try { + await addSnapshotProfileIndexes(f.k) + const created = f.queries.filter(query => query.sql.startsWith('CREATE TRIGGER')).length + await addSnapshotProfileIndexes(f.k) + expect(f.queries.filter(query => query.sql.startsWith('CREATE TRIGGER'))).toHaveLength(created) + expect(f.keys).toHaveLength(16) + } finally { + await f.k.destroy() + } +}) + +test('pool-only MySQL uses its actual database and preserves the explicit custom journal schema', async () => { + const f = mysqlFixture() + try { + expect(await readSnapshotProfileIndexState(f.k)).toBe(false) + expect(f.queries[0]).toEqual({ + sql: 'select * from information_schema.tables where table_name = ? and table_schema = database()', + values: ['knex_migrations'] + }) + expect(await readSnapshotProfileIndexState(f.k, { tableName: 'custom_journal', schemaName: 'custom_schema' })).toBe( + false + ) + expect(f.queries.at(-1)).toEqual({ + sql: 'select * from information_schema.tables where table_name = ? and table_schema = ?', + values: ['custom_journal', 'custom_schema'] + }) + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.test.ts new file mode 100644 index 000000000..beabc557e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotProfileIndexes.test.ts @@ -0,0 +1,273 @@ +import { knex, type Knex } from 'knex' +import { runInSeries } from '../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { + addSnapshotProfileIndexes, + removeSnapshotProfileIndexes, + snapshotProfileTables, + readSnapshotProfileIndexState, + SNAPSHOT_PROFILE_INDEX_MIGRATION +} from '../schema/snapshotProfileIndexMigration' + +const databases: Knex[] = [] +async function minimal(): Promise { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + databases.push(k) + await runInSeries(snapshotProfileTables, async ({ table, key }) => { + await k.schema.createTable(table, columns => { + columns.integer(key).primary() + columns.integer('userId').notNullable() + columns.text('value') + }) + }) + return k +} +afterEach(async () => { + jest.restoreAllMocks() + await runInSeries(databases.splice(0), k => k.destroy()) +}) + +test('bootstrap checkpoints exactly256rows and a failed next page resumes without losing independent low-ID writes', async () => { + const k = await minimal() + await runInSeries([0, 1, 2], async batch => { + await k('transactions').insert( + Array.from({ length: 200 }, (_, offset) => ({ + transactionId: batch * 200 + offset + 1, + userId: 1, + value: 'original' + })) + ) + }) + let reads = 0 + const failure = new Error('synthetic second page interruption') + const interrupt = (query: { sql: string }): void => { + if (query.sql.startsWith('select `transactionId`, `userId` from `transactions`') && ++reads === 2) throw failure + } + k.on('query', interrupt) + try { + await expect(addSnapshotProfileIndexes(k)).rejects.toBe(failure) + } finally { + k.off('query', interrupt) + } + expect(await k('snapshot_profile_index_progress').where('snapshotTableId', 0).first()).toEqual({ + snapshotTableId: 0, + afterRowId: 256, + complete: 0 + }) + expect(await k('snapshot_profile_keys')).toHaveLength(256) + await k('transactions').where('transactionId', 1).update({ userId: 2 }) + await k('transactions').where('transactionId', 2).delete() + await k('transactions').insert({ transactionId: 2, userId: 3, value: 'reinserted' }) + await addSnapshotProfileIndexes(k) + await addSnapshotProfileIndexes(k) + expect(await k('snapshot_profile_keys').orderBy('snapshotRowId')).toEqual( + (await k('transactions').orderBy('transactionId')).map(row => ({ + snapshotTableId: 0, + snapshotUserId: row.userId, + snapshotRowId: row.transactionId + })) + ) + expect(await k('snapshot_profile_index_progress').where('complete', 1)).toHaveLength(8) +}) + +test.each([0, 1, 255, 256, 257])('bootstrap preserves its exact final key for a source with %p rows', async count => { + const k = await minimal() + if (count > 0) + await k('tx_labels').insert(Array.from({ length: count }, (_, index) => ({ txLabelId: index + 1, userId: 1 }))) + await addSnapshotProfileIndexes(k) + expect(await k('snapshot_profile_keys')).toHaveLength(count) + expect(await k('snapshot_profile_index_progress').where('snapshotTableId', 3).first()).toEqual({ + snapshotTableId: 3, + afterRowId: count, + complete: 1 + }) +}) + +test('mismatched source triggers refuse both adoption and removal without deleting the foreign definition', async () => { + const k = await minimal() + await addSnapshotProfileIndexes(k) + await k.raw('DROP TRIGGER snapshot_profile_0_delete') + const sql = 'CREATE TRIGGER snapshot_profile_0_delete AFTER DELETE ON transactions BEGIN SELECT 1; END' + await k.raw(sql) + await expect(addSnapshotProfileIndexes(k)).rejects.toThrow('trigger definition mismatch') + await expect(removeSnapshotProfileIndexes(k)).rejects.toThrow('trigger definition mismatch') + expect(await k('sqlite_master').where({ type: 'trigger', name: 'snapshot_profile_0_delete' }).first('sql')).toEqual({ + sql + }) + expect(await k.schema.hasTable('snapshot_profile_keys')).toBe(true) +}) + +test('removing auxiliary state twice preserves every standard table/index definition and row', async () => { + const k = await minimal() + await k('transactions').insert({ transactionId: 5, userId: 2, value: 'preserved' }) + const definitions = async () => + await k('sqlite_master') + .whereIn('type', ['table', 'index']) + .select('type', 'name', 'tbl_name', 'sql') + .orderBy('name') + const before = await definitions() + await addSnapshotProfileIndexes(k) + await removeSnapshotProfileIndexes(k) + await removeSnapshotProfileIndexes(k) + expect(await definitions()).toEqual(before) + expect(await k('transactions')).toEqual([{ transactionId: 5, userId: 2, value: 'preserved' }]) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) +}) + +test('a wrong auxiliary primary key rejects before installing triggers or marking bootstrap complete', async () => { + const k = await minimal() + await k.schema.createTable('snapshot_profile_keys', columns => { + columns.integer('snapshotTableId').notNullable() + columns.integer('snapshotUserId').notNullable() + columns.integer('snapshotRowId').notNullable() + columns.primary(['snapshotUserId', 'snapshotRowId']) + }) + await expect(addSnapshotProfileIndexes(k)).rejects.toBeInstanceOf(WERR_INVALID_OPERATION) + expect(await k('snapshot_profile_keys')).toEqual([]) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) + expect(await k.schema.hasTable('snapshot_profile_index_progress')).toBe(false) +}) + +test('only an exact journaled migration with every completed table position selects the indexed path', async () => { + const k = await minimal() + expect(await readSnapshotProfileIndexState(k)).toBe(false) + await addSnapshotProfileIndexes(k) + expect(await readSnapshotProfileIndexState(k)).toBe(false) + await k.schema.createTable('custom_journal', columns => { + columns.increments('id') + columns.string('name').notNullable() + }) + const config = { tableName: 'custom_journal', schemaName: 'main' } + await k('custom_journal').insert({ name: 'future unrelated migration' }) + expect(await readSnapshotProfileIndexState(k, config)).toBe(false) + await k('custom_journal').insert({ name: SNAPSHOT_PROFILE_INDEX_MIGRATION }) + expect(await readSnapshotProfileIndexState(k, config)).toBe(true) + await k('snapshot_profile_index_progress').where('snapshotTableId', 3).update({ complete: 0 }) + await expect(readSnapshotProfileIndexState(k, config)).rejects.toThrow('migration is incomplete') + await k('snapshot_profile_index_progress').where('snapshotTableId', 3).update({ complete: 1, afterRowId: -1 }) + await expect(readSnapshotProfileIndexState(k, config)).rejects.toThrow('migration is incomplete') + await k('snapshot_profile_index_progress').where('snapshotTableId', 3).delete() + await expect(readSnapshotProfileIndexState(k, config)).rejects.toThrow('migration is incomplete') +}) + +test.each([ + [ + 'missing column', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId INTEGER NOT NULL, snapshotUserId INTEGER NOT NULL, PRIMARY KEY(snapshotTableId, snapshotUserId))' + ], + [ + 'wrong type', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId TEXT NOT NULL, snapshotUserId INTEGER NOT NULL, snapshotRowId INTEGER NOT NULL, PRIMARY KEY(snapshotTableId, snapshotUserId, snapshotRowId))' + ], + [ + 'nullable owner', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId INTEGER NOT NULL, snapshotUserId INTEGER, snapshotRowId INTEGER NOT NULL, PRIMARY KEY(snapshotTableId, snapshotUserId, snapshotRowId))' + ], + [ + 'unexpected default', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId INTEGER NOT NULL DEFAULT 0, snapshotUserId INTEGER NOT NULL, snapshotRowId INTEGER NOT NULL, PRIMARY KEY(snapshotTableId, snapshotUserId, snapshotRowId))' + ], + [ + 'reordered primary key', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId INTEGER NOT NULL, snapshotUserId INTEGER NOT NULL, snapshotRowId INTEGER NOT NULL, PRIMARY KEY(snapshotUserId, snapshotTableId, snapshotRowId))' + ], + [ + 'extra unique constraint', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId INTEGER NOT NULL, snapshotUserId INTEGER NOT NULL UNIQUE, snapshotRowId INTEGER NOT NULL, PRIMARY KEY(snapshotTableId, snapshotUserId, snapshotRowId))' + ], + [ + 'descending key', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId INTEGER NOT NULL, snapshotUserId INTEGER NOT NULL, snapshotRowId INTEGER NOT NULL, PRIMARY KEY(snapshotTableId DESC, snapshotUserId, snapshotRowId))' + ], + [ + 'different collation', + 'CREATE TABLE snapshot_profile_keys (snapshotTableId INTEGER NOT NULL, snapshotUserId INTEGER NOT NULL, snapshotRowId INTEGER NOT NULL, PRIMARY KEY(snapshotTableId COLLATE NOCASE, snapshotUserId, snapshotRowId))' + ] +])('auxiliary %s schema cannot be adopted or deleted', async (_name, sql) => { + const k = await minimal() + await k.raw(sql) + await expect(addSnapshotProfileIndexes(k)).rejects.toThrow('table definition mismatch') + await expect(removeSnapshotProfileIndexes(k)).rejects.toThrow('table definition mismatch') + expect(await k('sqlite_master').where({ type: 'table', name: 'snapshot_profile_keys' }).first('sql')).toEqual({ sql }) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) +}) + +test.each([ + { afterRowId: -1 }, + { afterRowId: 1.5 }, + { afterRowId: 'invalid' }, + { afterRowId: Number.MAX_SAFE_INTEGER + 1 }, + { complete: 2 }, + { complete: 'invalid' } +])('malformed bootstrap progress %j refuses before source reads', async change => { + const k = await minimal() + await addSnapshotProfileIndexes(k) + await k('snapshot_profile_index_progress') + .where('snapshotTableId', 0) + .update({ complete: 0, ...change }) + const queries: string[] = [] + const listen = (query: { sql: string }): void => { + queries.push(query.sql) + } + k.on('query', listen) + try { + await expect(addSnapshotProfileIndexes(k)).rejects.toThrow('bootstrap position') + } finally { + k.off('query', listen) + } + expect(queries.some(sql => sql.startsWith('select `transactionId`, `userId` from `transactions`'))).toBe(false) + expect(await k('snapshot_profile_keys')).toEqual([]) +}) + +test.each([0, -1, 1.5, 'invalid', Number.MAX_SAFE_INTEGER + 1])( + 'source profile key %p refuses before a bootstrap checkpoint can be completed', + async userId => { + const k = await minimal() + await k('transactions').insert({ transactionId: 1, userId }) + await expect(addSnapshotProfileIndexes(k)).rejects.toThrow('source key') + expect(await k('snapshot_profile_keys')).toEqual([]) + expect(await k('snapshot_profile_index_progress').where('snapshotTableId', 0).first()).toEqual({ + snapshotTableId: 0, + afterRowId: 0, + complete: 0 + }) + } +) + +test('journaled missing key/progress tables refuse rather than presenting an empty wallet', async () => { + const k = await minimal() + await k.schema.createTable('knex_migrations', columns => { + columns.string('name') + }) + await k('knex_migrations').insert({ name: SNAPSHOT_PROFILE_INDEX_MIGRATION }) + await expect(readSnapshotProfileIndexState(k)).rejects.toThrow('migration is incomplete') + await addSnapshotProfileIndexes(k) + await k.schema.dropTable('snapshot_profile_index_progress') + await expect(readSnapshotProfileIndexState(k)).rejects.toThrow('migration is incomplete') +}) + +test('incomplete SQLite primary-index metadata refuses before any bootstrap or trigger is installed', async () => { + const k = await minimal() + let omitted = false + const omitIndexMetadata = (rows: unknown, query: { sql: string }): void => { + if (query.sql.startsWith('PRAGMA index_list(') && query.sql.includes('snapshot_profile_keys')) { + expect(Array.isArray(rows)).toBe(true) + ;(rows as unknown[]).splice(0) + omitted = true + } + } + k.on('query-response', omitIndexMetadata) + try { + await expect(addSnapshotProfileIndexes(k)).rejects.toThrow('table definition mismatch') + } finally { + k.off('query-response', omitIndexMetadata) + } + expect(omitted).toBe(true) + expect(await k.schema.hasTable('snapshot_profile_index_progress')).toBe(false) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.integration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.integration.test.ts new file mode 100644 index 000000000..5eaca3df3 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.integration.test.ts @@ -0,0 +1,217 @@ +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { runInSeries } from '../../utility/runInSeries' +import { seedArchiveClosure } from '../../../test/utils/snapshotArchiveFixtures' +import { snapshotArchiveTables } from './archive/SnapshotArchive' +import { openKnexSnapshotArchiveSource, type SnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import { + addSnapshotRelationIndexes, + removeSnapshotRelationIndexes, + SNAPSHOT_RELATION_INDEX_MIGRATION +} from '../schema/snapshotRelationIndexMigration' + +const identity = '02' + '11'.repeat(32) +const stores: StorageKnex[] = [] +const writers: Knex[] = [] +const directories: string[] = [] +const dates = { created_at: '2026-01-01T00:00:00.000Z', updated_at: '2026-01-01T00:00:00.000Z' } +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-relation-index-')) + directories.push(directory) + const options = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + } + const k = knex(options) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + stores.push(source) + await k.raw('PRAGMA journal_mode=WAL') + await migrateBeforeSqliteGeneration(source, 'relation index fixture', 'synthetic-relation-index') + await source.makeAvailable() + // Start with the immediately preceding schema on both old and new source. + await removeSnapshotRelationIndexes(k) + await k('knex_migrations').where('name', SNAPSHOT_RELATION_INDEX_MIGRATION).delete() + const { user } = await source.findOrInsertUser(identity) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, other.userId) + const writer = knex(options) + writers.push(writer) + return { source, writer, k, userId: user.userId, otherId: other.userId } +} +async function journal(k: Knex): Promise { + await k('knex_migrations').insert({ name: SNAPSHOT_RELATION_INDEX_MIGRATION, batch: 99, migration_time: new Date() }) +} +async function pages(view: WalletReadSnapshot | SnapshotArchiveSource) { + expect(Object.hasOwn(view, 'relationIndexes')).toBe(false) + if ('validateClosure' in view) await view.validateClosure() + const result: Record = {} + await runInSeries(snapshotArchiveTables, async table => { + let cursor: WalletSnapshotCursor | undefined + const selected: unknown[] = [] + let complete = false + for (let pageNumber = 0; pageNumber < 25 && !complete; pageNumber++) { + const page = await view.readPage(table, cursor, { maxRows: 1, maxBytes: 131072 }) + selected.push({ rows: page.rows, after: page.cursor?.after, payloadBytes: page.payloadBytes, done: page.done }) + complete = page.done + cursor = page.cursor + } + expect(complete).toBe(true) + result[table] = selected + }) + return result +} +afterEach(async () => { + await runInSeries(writers.splice(0), k => k.destroy()) + await runInSeries(stores.splice(0), source => source.destroy()) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('indexed ordinary/archive pages preserve all13tables and original source indexes/legacy offsets', async () => { + const { source, k, userId } = await fixture() + const legacy = async () => { + const all: Record = {} + await runInSeries( + [ + ['findTransactions', 'transactionId'], + ['findOutputs', 'outputId'], + ['findCertificates', 'certificateId'], + ['findTxLabels', 'txLabelId'], + ['findOutputBaskets', 'basketId'], + ['findOutputTags', 'outputTagId'], + ['findCommissions', 'commissionId'], + ['findSyncStates', 'syncStateId'] + ] as const, + async ([method, key]) => { + all[method] = [] + await runInSeries([0, 1], async offset => { + const rows = await source[method]({ partial: { userId }, paged: { limit: 1, offset } }) + all[method].push(rows.map(row => (row as unknown as Record)[key])) + }) + } + ) + return all + } + const standard = async () => + await k('sqlite_master') + .whereIn('type', ['table', 'index']) + .whereNotIn('tbl_name', ['snapshot_relation_keys', 'snapshot_relation_index_progress']) + .select('type', 'name', 'tbl_name', 'sql') + .orderBy('name') + const originalIndexes = await standard() + const originalOffsets = await legacy() + const old = await source.openWalletReadSnapshot(identity) + let baseline + try { + baseline = await pages(old) + } finally { + await old.close() + } + await addSnapshotRelationIndexes(k) + expect(await standard()).toEqual(originalIndexes) + expect(await legacy()).toEqual(originalOffsets) + const queries: Array<{ sql: string; bindings: Knex.RawBinding[] }> = [] + const listen = (query: { sql: string; bindings: Knex.RawBinding[] }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_relation_keys') + ) + queries.push(query) + } + k.on('query', listen) + try { + const partial = await source.openWalletReadSnapshot(identity) + try { + expect(await pages(partial)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await partial.close() + } + await journal(k) + await runInSeries( + [() => source.openWalletReadSnapshot(identity), () => openKnexSnapshotArchiveSource(source, identity)], + async open => { + const view = await open() + try { + expect(await pages(view)).toEqual(baseline) + } finally { + await view.close() + } + } + ) + expect(queries.length).toBeGreaterThan(0) + } finally { + k.off('query', listen) + } + // Explain only page queries; closure may legitimately visit its relational parents. + await runInSeries( + queries.filter(query => query.sql.includes('__snapshotBytes') || query.sql.includes('.*')), + async query => { + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + query.sql, query.bindings) + expect(plan.some(row => row.detail.includes('SEARCH snapshot_relation_keys USING COVERING INDEX'))).toBe(true) + expect(plan.some(row => /SCAN |TEMP B-TREE/.test(row.detail))).toBe(false) + } + ) +}) + +test.each(['ordinary', 'archive'] as const)( + '%s mode and all13table pages stay bound across independent journal/profile/progress writes', + async kind => { + const { source, k, writer, userId, otherId } = await fixture() + await addSnapshotRelationIndexes(k) + const open = async () => + kind === 'ordinary' + ? await source.openWalletReadSnapshot(identity) + : await openKnexSnapshotArchiveSource(source, identity) + const queries: string[] = [] + const listen = (query: { sql: string }): void => { + if ( + query.sql.startsWith('select') && + query.sql.includes('cross join') && + query.sql.includes('snapshot_relation_keys') + ) + queries.push(query.sql) + } + k.on('query', listen) + const old = await open() + let baseline + try { + baseline = await pages(old) + queries.length = 0 + await journal(writer) + expect(await pages(old)).toEqual(baseline) + expect(queries).toEqual([]) + } finally { + await old.close() + } + const indexed = await open() + try { + await writer.transaction(async trx => { + await trx('snapshot_relation_index_progress').where('snapshotTableId', 0).update({ complete: 0 }) + await trx('tx_labels').where('txLabelId', 3).update({ userId: otherId, label: 'moved' }) + await trx('tx_labels').insert({ ...dates, txLabelId: 4, userId, label: 'new after view', isDeleted: false }) + }) + expect(await pages(indexed)).toEqual(baseline) + expect(queries.length).toBeGreaterThan(0) + } finally { + await indexed.close() + k.off('query', listen) + } + await expect(open()).rejects.toThrow('migration is incomplete') + await writer('snapshot_relation_index_progress').where('snapshotTableId', 0).update({ complete: 1 }) + const fresh = await open() + try { + expect((await fresh.readPage('txLabels')).rows.map(row => row.txLabelId)).toEqual([1, 4]) + } finally { + await fresh.close() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.mysql.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.mysql.test.ts new file mode 100644 index 000000000..85e7cf478 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.mysql.test.ts @@ -0,0 +1,373 @@ +import { knex } from 'knex' +import { + addSnapshotRelationIndexes, + removeSnapshotRelationIndexes, + readSnapshotRelationIndexState, + SNAPSHOT_RELATION_INDEX_MIGRATION, + snapshotNumericRelations +} from '../schema/snapshotRelationIndexMigration' + +type MetadataKind = 'columns' | 'indexes' | 'triggers' +type Metadata = Array> + +/** Installed MySQL compiler and transaction protocol; native fixtures separately prove database semantics. */ +function mysqlFixture(change: (kind: MetadataKind, rows: Metadata) => unknown = (_kind, rows) => rows) { + const k = knex({ client: 'mysql2' }) + const queries: Array<{ sql: string; values: unknown[] }> = [] + const tables = new Set(['knex_migrations']) + const indexes = new Map() + const triggers = new Map>() + const progress = new Map< + number, + { snapshotTableId: number; afterLeftId: number; afterRightId: number; complete: boolean } + >() + const keys: Array<{ + snapshotTableId: number + snapshotUserId: number + snapshotLeftId: number + snapshotRightId: number + snapshotMembership: number + }> = [] + let journaled = false + const answer = (sql: string, values: unknown[]): unknown => { + queries.push({ sql, values }) + if (sql.startsWith('select * from information_schema.tables')) + return tables.has(String(values[0])) ? [{ TABLE_NAME: values[0] }] : [] + if (sql.startsWith('create table')) { + tables.add(sql.match(/^create table `([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('drop table')) { + tables.delete(sql.match(/`([^`]+)`/)![1]) + return [] + } + if (sql.startsWith('SELECT COLUMN_NAME')) { + const isKeys = values[0] === 'snapshot_relation_keys' + const names = isKeys + ? ['snapshotTableId', 'snapshotUserId', 'snapshotLeftId', 'snapshotRightId', 'snapshotMembership'] + : ['snapshotTableId', 'afterLeftId', 'afterRightId', 'complete'] + const types = isKeys + ? ['int', 'int unsigned', 'int unsigned', 'int unsigned', 'int unsigned'] + : ['int', 'int unsigned', 'int unsigned', 'tinyint'] + return change( + 'columns', + names.map((name, i) => ({ name, type: types[i], nullable: 'NO', defaultValue: null, extra: '' })) + ) + } + if (sql.startsWith('alter table `snapshot_relation_keys` add index')) { + const name = sql.match(/add index `([^`]+)`/)![1] + indexes.set( + name, + [...sql.matchAll(/`([^`]+)`/g)].slice(2).map(match => match[1]) + ) + return [] + } + if (sql.startsWith('SELECT INDEX_NAME FROM')) + return indexes.has(String(values[1])) ? [{ INDEX_NAME: values[1] }] : [] + if (sql.startsWith('SELECT INDEX_NAME AS')) { + const primary = + values[0] === 'snapshot_relation_keys' + ? ['snapshotTableId', 'snapshotUserId', 'snapshotLeftId', 'snapshotRightId'] + : ['snapshotTableId'] + return change('indexes', [ + ...primary.map(columnName => ({ name: 'PRIMARY', columnName, nonUnique: 0, direction: 'A', prefix: null })), + ...(values[0] === 'snapshot_relation_keys' + ? [...indexes].flatMap(([name, columns]) => + columns.map(columnName => ({ name, columnName, nonUnique: 1, direction: 'A', prefix: null })) + ) + : []) + ]) + } + if (sql.startsWith('SELECT EVENT_MANIPULATION')) + return change('triggers', triggers.has(String(values[0])) ? [triggers.get(String(values[0]))!] : []) + if (sql.startsWith('CREATE TRIGGER')) { + const [, name, timing, event, tableName, body] = sql.match( + /^CREATE TRIGGER (\w+) (BEFORE|AFTER) (\w+) ON (\w+) FOR EACH ROW (.*)$/ + )! + triggers.set(name, { event, timing, tableName, body }) + return [] + } + if (sql.startsWith('DROP TRIGGER')) { + triggers.delete(sql.split(' ').at(-1)!.replaceAll('`', '')) + return [] + } + if (sql.startsWith('insert ignore into `snapshot_relation_index_progress`')) { + const [afterLeftId, afterRightId, complete, snapshotTableId] = values as [number, number, boolean, number] + if (!progress.has(snapshotTableId)) + progress.set(snapshotTableId, { snapshotTableId, afterLeftId, afterRightId, complete }) + return { affectedRows: 1, insertId: 0 } + } + if (sql.startsWith('select * from `snapshot_relation_index_progress`')) { + expect(sql).toMatch(/for update$/) + const row = progress.get(Number(values[0])) + return row === undefined ? [] : [{ ...row }] + } + if (sql.startsWith('select `snapshotTableId`, `afterLeftId`, `afterRightId`, `complete`')) + return [...progress.values()].map(row => ({ ...row })).slice(0, Number(values[0])) + if (sql.startsWith('update `snapshot_relation_index_progress`')) { + const [afterLeftId, afterRightId, complete, tableId] = values as [number, number, boolean, number] + Object.assign(progress.get(tableId)!, { afterLeftId, afterRightId, complete }) + return { affectedRows: 1 } + } + if (sql.startsWith('insert into `snapshot_relation_keys`')) { + const fields = sql + .slice(sql.indexOf('(') + 1, sql.indexOf(')')) + .split(', ') + .map(value => value.replaceAll('`', '')) + for (let i = 0; i < values.length - 1; i += fields.length) { + const row = Object.fromEntries( + fields.map((field, offset) => [field, Number(values[i + offset])]) + ) as (typeof keys)[number] + const existing = keys.find( + key => + key.snapshotTableId === row.snapshotTableId && + key.snapshotUserId === row.snapshotUserId && + key.snapshotLeftId === row.snapshotLeftId && + key.snapshotRightId === row.snapshotRightId + ) + if (existing === undefined) keys.push(row) + else existing.snapshotMembership |= Number(values.at(-1)) + } + return { affectedRows: 1 } + } + if (sql.startsWith('select `name` from `knex_migrations`')) + return journaled ? [{ name: SNAPSHOT_RELATION_INDEX_MIGRATION }] : [] + if (/^(BEGIN|COMMIT|ROLLBACK)/.test(sql)) return [] + const source = snapshotNumericRelations.find(({ table }) => sql.includes('from `' + table + '`')) + if (source !== undefined) { + expect(sql).toMatch(/for update$/) + expect(values.at(-1)).toBe(256) + expect(sql).toContain(' OR ') + return [ + { [source.leftKey]: 1, [source.rightKey]: 1 }, + { [source.leftKey]: 3, [source.rightKey]: 2 } + ].filter(row => row[source.leftKey] > Number(values[0])) + } + const parent = snapshotNumericRelations + .flatMap(p => [ + [p.left, p.leftKey], + [p.right, p.rightKey] + ]) + .find(([table]) => sql.includes('from `' + table + '`')) + if (parent !== undefined) { + expect(sql).toMatch(/lock in share mode$/) + return values.map(id => ({ [parent[1]]: Number(id), userId: Number(id) === 1 ? 1 : 2 })) + } + throw new Error('Unexpected synthetic driver query: ' + sql) + } + const connection = { + query( + query: { sql: string }, + values: unknown[], + callback: (error: Error | null, rows?: unknown, fields?: unknown[]) => void + ) { + try { + callback(null, answer(query.sql, values), []) + } catch (error) { + callback(error as Error) + } + } + } + jest.spyOn(k.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(k.client, 'releaseConnection').mockResolvedValue(undefined) + return { + k, + queries, + keys, + progress, + triggers, + tables, + journal: () => { + journaled = true + } + } +} + +afterEach(() => jest.restoreAllMocks()) + +test('MySQL DDL and locked bounded bootstrap preserve completion ordering and support repeat/removal', async () => { + const f = mysqlFixture() + try { + await addSnapshotRelationIndexes(f.k) + expect(f.triggers.size).toBe(24) + expect(f.keys).toEqual( + snapshotNumericRelations.flatMap((_table, snapshotTableId) => [ + { snapshotTableId, snapshotUserId: 1, snapshotLeftId: 1, snapshotRightId: 1, snapshotMembership: 3 }, + { snapshotTableId, snapshotUserId: 2, snapshotLeftId: 3, snapshotRightId: 2, snapshotMembership: 3 } + ]) + ) + expect([...f.progress.values()]).toEqual( + snapshotNumericRelations.map((_table, snapshotTableId) => ({ + snapshotTableId, + afterLeftId: 3, + afterRightId: 2, + complete: true + })) + ) + expect( + f.queries.filter(query => query.sql.includes('for update') && !query.sql.includes('snapshot_relation')) + ).toHaveLength(2) + expect(f.queries.find(query => query.sql.startsWith('create table `snapshot_relation_keys`'))!.sql).toContain( + 'primary key (`snapshotTableId`, `snapshotUserId`, `snapshotLeftId`, `snapshotRightId`)' + ) + expect(f.queries.filter(query => query.sql === 'BEGIN;')).toHaveLength(2) + expect(f.queries.filter(query => query.sql === 'COMMIT;')).toHaveLength(2) + f.journal() + expect(await readSnapshotRelationIndexState(f.k)).toBe(true) + await addSnapshotRelationIndexes(f.k) + expect(f.keys).toHaveLength(4) + await removeSnapshotRelationIndexes(f.k) + expect(f.triggers.size).toBe(0) + expect(f.tables.has('snapshot_relation_keys')).toBe(false) + } finally { + await f.k.destroy() + } +}) + +test.each([ + ['columns', []], + ['columns', null], + ['indexes', []], + ['indexes', null] +] as const)('MySQL missing/non-array %s metadata refuses before installing triggers', async (kind, replacement) => { + const f = mysqlFixture((current, rows) => (current === kind ? replacement : rows)) + try { + await expect(addSnapshotRelationIndexes(f.k)).rejects.toThrow('table definition mismatch') + expect(f.triggers.size).toBe(0) + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } +}) + +test.each([ + { name: 'unexpected' }, + { type: 'bigint' }, + { nullable: 'YES' }, + { defaultValue: 0 }, + { extra: 'auto_increment' } +])('MySQL column metadata mismatch %j refuses without bootstrap', async changed => { + const f = mysqlFixture((kind, rows) => (kind === 'columns' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows)) + try { + await expect(addSnapshotRelationIndexes(f.k)).rejects.toThrow('table definition mismatch') + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } +}) + +test.each([{ direction: 'D' }, { prefix: 1 }, { columnName: 'unexpected' }])( + 'MySQL primary-key mismatch %j refuses adoption', + async changed => { + const f = mysqlFixture((kind, rows) => (kind === 'indexes' ? [{ ...rows[0], ...changed }, ...rows.slice(1)] : rows)) + try { + await expect(addSnapshotRelationIndexes(f.k)).rejects.toThrow('table definition mismatch') + } finally { + await f.k.destroy() + } + } +) + +test('MySQL extra unique constraints refuse while nonunique indexes remain compatible', async () => { + for (const nonUnique of [0, 1]) { + const f = mysqlFixture((kind, rows) => + kind === 'indexes' ? [...rows, { name: 'extra', columnName: 'snapshotRowId', nonUnique }] : rows + ) + try { + if (nonUnique === 0) await expect(addSnapshotRelationIndexes(f.k)).rejects.toThrow('table definition mismatch') + else { + await addSnapshotRelationIndexes(f.k) + expect(f.keys).toHaveLength(4) + } + } finally { + await f.k.destroy() + } + } +}) + +test('a MySQL migration in an outer transaction refuses before DDL can commit it implicitly', async () => { + const f = mysqlFixture() + try { + await f.k.transaction(async trx => { + await expect(addSnapshotRelationIndexes(trx)).rejects.toThrow('independent DDL') + await expect(removeSnapshotRelationIndexes(trx)).rejects.toThrow('independent DDL') + }) + expect(f.queries.every(query => /^(BEGIN|COMMIT)/.test(query.sql))).toBe(true) + } finally { + await f.k.destroy() + } +}) + +test.each([ + { event: 'UNKNOWN' }, + { timing: 'BEFORE' }, + { tableName: 'foreign_table' }, + { body: 'BEGIN SELECT 1; END' } +])('MySQL stored trigger mismatch %j refuses adoption/removal without deleting definitions', async changed => { + let corrupt = false + const f = mysqlFixture((kind, rows) => (corrupt && kind === 'triggers' ? [{ ...rows[0], ...changed }] : rows)) + try { + await addSnapshotRelationIndexes(f.k) + corrupt = true + await expect(addSnapshotRelationIndexes(f.k)).rejects.toThrow('trigger definition mismatch') + await expect(removeSnapshotRelationIndexes(f.k)).rejects.toThrow('trigger definition mismatch') + expect(f.triggers.size).toBe(24) + expect(f.tables.has('snapshot_relation_keys')).toBe(true) + } finally { + await f.k.destroy() + } +}) + +test.each([null, [undefined], [{}, {}]])( + 'MySQL malformed trigger metadata %p refuses before creation', + async replacement => { + const f = mysqlFixture((kind, rows) => (kind === 'triggers' ? replacement : rows)) + try { + await expect(addSnapshotRelationIndexes(f.k)).rejects.toThrow(/trigger (metadata|definition mismatch)/) + expect(f.triggers.size).toBe(0) + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } + } +) + +test('MySQL native-width metadata and harmless body whitespace do not require recreation', async () => { + const f = mysqlFixture((kind, rows) => + kind === 'columns' + ? rows.map(row => ({ ...row, type: String(row.type).replace('int', 'int(11)') })) + : kind === 'triggers' + ? rows.map(row => ({ ...row, body: String(row.body).replace(/ /g, ' \n ') })) + : rows + ) + try { + await addSnapshotRelationIndexes(f.k) + const created = f.queries.filter(query => query.sql.startsWith('CREATE TRIGGER')).length + await addSnapshotRelationIndexes(f.k) + expect(f.queries.filter(query => query.sql.startsWith('CREATE TRIGGER'))).toHaveLength(created) + expect(f.keys).toHaveLength(4) + } finally { + await f.k.destroy() + } +}) + +test('pool-only MySQL uses its actual database and preserves the explicit custom journal schema', async () => { + const f = mysqlFixture() + try { + expect(await readSnapshotRelationIndexState(f.k)).toBe(false) + expect(f.queries[0]).toEqual({ + sql: 'select * from information_schema.tables where table_name = ? and table_schema = database()', + values: ['knex_migrations'] + }) + expect( + await readSnapshotRelationIndexState(f.k, { tableName: 'custom_journal', schemaName: 'custom_schema' }) + ).toBe(false) + expect(f.queries.at(-1)).toEqual({ + sql: 'select * from information_schema.tables where table_name = ? and table_schema = ?', + values: ['custom_journal', 'custom_schema'] + }) + expect(f.keys).toEqual([]) + } finally { + await f.k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.test.ts new file mode 100644 index 000000000..a29630792 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotRelationIndexes.test.ts @@ -0,0 +1,256 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../utility/runInSeries' +import { expectRelationMembership, minimalRelationDatabase } from '../../../test/utils/snapshotRelationFixtures' +import { + addSnapshotRelationIndexes as install, + removeSnapshotRelationIndexes as remove, + readSnapshotRelationIndexState as enabled, + SNAPSHOT_RELATION_INDEX_MIGRATION as migration +} from '../schema/snapshotRelationIndexMigration' + +const databases: Knex[] = [] +async function fixture(): Promise { + const k = await minimalRelationDatabase() + databases.push(k) + return k +} +afterEach(async () => { + jest.restoreAllMocks() + await runInSeries(databases.splice(0), k => k.destroy()) +}) + +test.each([0, 1, 255, 256, 257])( + 'bootstrap resumes the composite cursor for %p mappings without altering source indexes', + async count => { + const k = await fixture() + await k('tx_labels').insert({ txLabelId: 1, userId: 1 }) + if (count) { + await k('transactions').insert( + Array.from({ length: count }, (_, i) => ({ transactionId: i + 1, userId: (i % 2) + 1 })) + ) + await k('tx_labels_map').insert( + Array.from({ length: count }, (_, i) => ({ txLabelId: 1, transactionId: i + 1, isDeleted: i % 2 })) + ) + } + const schema = () => + k('sqlite_master').whereIn('type', ['table', 'index']).select('name', 'type', 'sql').orderBy('name') + const before = await schema() + await install(k) + await install(k) + await expectRelationMembership(k) + expect(await k('snapshot_relation_index_progress').where('snapshotTableId', 0).first()).toEqual({ + snapshotTableId: 0, + afterLeftId: count ? 1 : 0, + afterRightId: count, + complete: 1 + }) + await remove(k) + await remove(k) + expect(await schema()).toEqual(before) + expect(await k('tx_labels_map')).toHaveLength(count) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) + } +) + +test('a committed bootstrap page resumes after failure and preserves lower-key writes and owner changes', async () => { + const k = await fixture() + await k('tx_labels').insert({ txLabelId: 1, userId: 1 }) + await k('transactions').insert(Array.from({ length: 300 }, (_, i) => ({ transactionId: i + 2, userId: 1 }))) + await k('tx_labels_map').insert( + Array.from({ length: 300 }, (_, i) => ({ txLabelId: 1, transactionId: i + 2, isDeleted: false })) + ) + const failure = new Error('synthetic relation bootstrap interruption') + let reads = 0 + const interrupt = (query: { sql: string }): void => { + if (query.sql.startsWith('select `txLabelId`, `transactionId` from `tx_labels_map`') && ++reads === 2) throw failure + } + k.on('query', interrupt) + try { + await expect(install(k)).rejects.toBe(failure) + } finally { + k.off('query', interrupt) + } + expect(await k('snapshot_relation_index_progress').where('snapshotTableId', 0).first()).toEqual({ + snapshotTableId: 0, + afterLeftId: 1, + afterRightId: 257, + complete: 0 + }) + expect(await k('snapshot_relation_keys')).toHaveLength(256) + await k('transactions').insert({ transactionId: 1, userId: 2 }) + await k('tx_labels_map').insert({ txLabelId: 1, transactionId: 1, isDeleted: true }) + await k('tx_labels').where('txLabelId', 1).update({ userId: 3 }) + await k('tx_labels_map').where('transactionId', 2).delete() + await install(k) + await expectRelationMembership(k) +}) + +test('membership retains both ownership bases, cross-profile and orphan visibility, and physical deletion semantics', async () => { + const k = await fixture() + await install(k) + await k('tx_labels').insert({ txLabelId: 1, userId: 1 }) + await k('transactions').insert({ transactionId: 1, userId: 1 }) + await k('tx_labels_map').insert({ txLabelId: 1, transactionId: 1, isDeleted: true }) + await expectRelationMembership(k) + expect((await k('snapshot_relation_keys').first()).snapshotMembership).toBe(3) + await k('tx_labels').where('txLabelId', 1).update({ userId: 2 }) + await expectRelationMembership(k) + expect(await k('snapshot_relation_keys')).toHaveLength(2) + await k('transactions').delete() + await expectRelationMembership(k) + expect((await k('snapshot_relation_keys').first()).snapshotMembership).toBe(1) + await k('tx_labels_map').delete() + await expectRelationMembership(k) + expect(await k('snapshot_relation_keys')).toEqual([]) +}) + +test('journal, complete positions and exact tables are required before selecting indexed reads', async () => { + const k = await fixture() + expect(await enabled(k)).toBe(false) + await install(k) + expect(await enabled(k)).toBe(false) + await k.schema.createTable('custom_journal', t => { + t.string('name') + }) + const config = { tableName: 'custom_journal', schemaName: 'main' } + await k('custom_journal').insert({ name: 'unrelated migration' }) + expect(await enabled(k, config)).toBe(false) + await k('custom_journal').insert({ name: migration }) + expect(await enabled(k, config)).toBe(true) + await k('snapshot_relation_index_progress').where('snapshotTableId', 1).update({ complete: 0 }) + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') + await k('snapshot_relation_index_progress').where('snapshotTableId', 1).update({ complete: 1, afterRightId: -1 }) + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') + await k('snapshot_relation_index_progress').where('snapshotTableId', 1).delete() + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') + await k.schema.dropTable('snapshot_relation_index_progress') + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') + await k.schema.dropTable('snapshot_relation_keys') + await expect(enabled(k, config)).rejects.toThrow('migration is incomplete') +}) + +test.each([ + { afterLeftId: -1 }, + { afterRightId: -1 }, + { afterLeftId: 1.5 }, + { afterRightId: 'invalid' }, + { afterLeftId: Number.MAX_SAFE_INTEGER + 1 }, + { complete: 2 } +])('invalid progress %j refuses before reading source maps', async change => { + const k = await fixture() + await install(k) + await k('snapshot_relation_index_progress') + .where('snapshotTableId', 0) + .update({ complete: 0, ...change }) + const read = jest.fn() + k.on('query', read) + await expect(install(k)).rejects.toThrow('bootstrap position') + expect(read.mock.calls.some(([q]) => q.sql.startsWith('select `txLabelId`, `transactionId` from'))).toBe(false) +}) + +test.each([0, -1, 1.5, 'invalid', Number.MAX_SAFE_INTEGER + 1])( + 'invalid parent owner %p rolls back its page and checkpoint', + async userId => { + const k = await fixture() + await k('tx_labels').insert({ txLabelId: 1, userId }) + await k('tx_labels_map').insert({ txLabelId: 1, transactionId: 1, isDeleted: false }) + await expect(install(k)).rejects.toThrow('source key') + expect(await k('snapshot_relation_keys')).toEqual([]) + expect(await k('snapshot_relation_index_progress').where('snapshotTableId', 0).first()).toEqual({ + snapshotTableId: 0, + afterLeftId: 0, + afterRightId: 0, + complete: 0 + }) + } +) + +test('conflicting trigger definitions refuse adoption and removal before dropping any observer or producer', async () => { + const k = await fixture() + await install(k) + await k.raw('DROP TRIGGER snapshot_relation_1_right_delete') + await k.raw('CREATE TRIGGER snapshot_relation_1_right_delete AFTER DELETE ON outputs BEGIN SELECT 1; END') + const before = await k('sqlite_master').where('type', 'trigger').orderBy('name') + await expect(install(k)).rejects.toThrow('trigger definition mismatch') + await expect(remove(k)).rejects.toThrow('trigger definition mismatch') + expect(await k('sqlite_master').where('type', 'trigger').orderBy('name')).toEqual(before) + expect(await k.schema.hasTable('snapshot_relation_keys')).toBe(true) +}) + +test.each([ + 'CREATE UNIQUE INDEX foreign_unique_relation ON snapshot_relation_keys(snapshotLeftId)', + 'DROP INDEX snapshot_relation_right', + 'CREATE INDEX foreign_partial_relation ON snapshot_relation_keys(snapshotRightId) WHERE snapshotMembership=1' +])('schema inspection preserves unrelated definitions: %s', async sql => { + const k = await fixture() + await install(k) + await k.raw(sql) + if (sql.startsWith('CREATE INDEX foreign_partial')) { + await install(k) + await expectRelationMembership(k) + } else if (sql.startsWith('DROP')) { + await expect(remove(k)).rejects.toThrow('table definition mismatch') + await install(k) + expect(await k('sqlite_master').where({ type: 'index', name: 'snapshot_relation_right' })).toHaveLength(1) + } else { + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') + expect(await k('sqlite_master').where({ type: 'index', name: 'foreign_unique_relation' })).toHaveLength(1) + } +}) + +test('partial, reversed or wrong-column maintenance indexes cannot silently replace required indexes', async () => { + const k = await fixture() + await install(k) + await k.raw('DROP INDEX snapshot_relation_right') + await k.raw( + 'CREATE INDEX snapshot_relation_right ON snapshot_relation_keys(snapshotTableId,snapshotUserId,snapshotRightId DESC,snapshotLeftId)' + ) + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') +}) + +test('a foreign auxiliary table shape refuses adoption and removal without changing its rows', async () => { + const k = await fixture() + await k.schema.createTable('snapshot_relation_keys', t => { + t.text('unrelated') + }) + await k('snapshot_relation_keys').insert({ unrelated: 'preserve' }) + await expect(install(k)).rejects.toThrow('table definition mismatch') + await expect(remove(k)).rejects.toThrow('table definition mismatch') + expect(await k('snapshot_relation_keys')).toEqual([{ unrelated: 'preserve' }]) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) +}) + +test('missing primary-index metadata refuses before installing triggers or publishing progress', async () => { + const k = await fixture() + const omit = (rows: unknown, query: { sql: string }): void => { + if (query.sql.startsWith('PRAGMA index_list(') && query.sql.includes('snapshot_relation_keys')) { + expect(Array.isArray(rows)).toBe(true) + ;(rows as unknown[]).splice(0) + } + } + k.on('query-response', omit) + try { + await expect(install(k)).rejects.toThrow('table definition mismatch') + } finally { + k.off('query-response', omit) + } + expect(await k.schema.hasTable('snapshot_relation_index_progress')).toBe(false) + expect(await k('sqlite_master').where('type', 'trigger')).toEqual([]) +}) + +test('bootstrap retains the existing owner of an inconsistent mapping and permits a missing ownership basis', async () => { + const k = await fixture() + await k('tx_labels').insert({ txLabelId: 1, userId: 1 }) + await k('tx_labels_map').insert([ + { txLabelId: 1, transactionId: 1, isDeleted: true }, + { txLabelId: 2, transactionId: 2, isDeleted: false } + ]) + await install(k) + await expectRelationMembership(k) + expect(await k('snapshot_relation_keys')).toEqual([ + { snapshotTableId: 0, snapshotUserId: 1, snapshotLeftId: 1, snapshotRightId: 1, snapshotMembership: 1 } + ]) + expect(await k('snapshot_relation_index_progress').where('complete', 1)).toHaveLength(2) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotResourceLimitError.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotResourceLimitError.ts new file mode 100644 index 000000000..19367f726 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotResourceLimitError.ts @@ -0,0 +1,4 @@ +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' + +/** A safe source/page admission limit, distinct from malformed data or a changed session. */ +export class SnapshotResourceLimitError extends WERR_INVALID_OPERATION {} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteActualSchema.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteActualSchema.test.ts new file mode 100644 index 000000000..b9165a96b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteActualSchema.test.ts @@ -0,0 +1,128 @@ +import { knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { names } from '../schema/snapshotSqliteIndexGeneration' +import { legacyNames } from '../schema/snapshotSqliteMembership' +import type { Knex } from 'knex' +import { exact, replace, tables } from '../../../test/utils/snapshotSqliteFixtures' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { seedArchiveClosure } from '../../../test/utils/snapshotArchiveFixtures' +import { snapshotArchiveTables } from './archive/SnapshotArchive' +import { walletSnapshotSourceQuery } from './KnexWalletReadSnapshot' +import { openKnexSnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import type { SnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' + +async function pages(view: WalletReadSnapshot | SnapshotArchiveSource) { + const result: Record = {} + try { + if ('validateClosure' in view) await view.validateClosure() + for (const table of snapshotArchiveTables) { + let cursor: WalletSnapshotCursor | undefined + const rows: unknown[] = [] + for (let n = 0; n < 30; n++) { + const page = await view.readPage(table, cursor, { maxRows: 1, maxBytes: 131072 }) + rows.push(...page.rows) + if (page.done) { + result[table] = rows + break + } + cursor = page.cursor + } + expect(result[table]).toBeDefined() + } + return result + } finally { + await view.close() + } +} + +test.each([false, true])( + 'actual migrated wallet preserves replacement membership and both readers, recursive=%s', + async recursive => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-identity-actual-')) + const k = knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + const identities = ['02' + '11'.repeat(32), '03' + '22'.repeat(32)] + try { + await k.raw('PRAGMA journal_mode=WAL') + await source.migrate('identity fixture', 'synthetic-identity-fixture') + await source.makeAvailable() + await k.raw('PRAGMA recursive_triggers=' + Number(recursive)) + const users = [] + for (const identity of identities) users.push((await source.findOrInsertUser(identity)).user.userId) + await seedArchiveClosure(source, users[0], users[1]) + const originalSource = await k('sqlite_master') + .whereIn('tbl_name', tables) + .whereIn('type', ['table', 'index']) + .orderBy('name') + const mapping = new Map( + Object.entries(legacyNames).map(([key, value]) => [value, names[key as keyof typeof names]]) + ) + const rebuilt = new Proxy(k, { + apply(target, self, args: unknown[]) { + if (typeof args[0] === 'string' && mapping.has(args[0])) args[0] = mapping.get(args[0]) + return Reflect.apply(target, self, args) + } + }) as Knex + expect( + await k('sqlite_master').whereIn('tbl_name', tables).whereIn('type', ['table', 'index']).orderBy('name') + ).toEqual(originalSource) + await exact(rebuilt) + // Move both source rows of one profile, retaining all PKs and foreign keys. + // Each REPLACE is legal with foreign_keys enabled; closure agrees at the end. + for (const table of tables) { + const rows = await k(table) + for (const row of rows) { + if ('userId' in row && row.userId === users[0]) row.userId = users[1] + await replace(k, table, row) + await exact(rebuilt) + } + } + expect((await k.raw('PRAGMA foreign_keys'))[0].foreign_keys).toBe(1) + expect(await k.raw('PRAGMA foreign_key_check')).toEqual([]) + for (const [index, userId] of users.entries()) { + for (const table of snapshotArchiveTables) { + const name = + ( + { + provenTxs: 'proven_txs', + provenTxReqs: 'proven_tx_reqs', + outputBaskets: 'output_baskets', + outputTags: 'output_tags', + outputTagMaps: 'output_tags_map', + txLabels: 'tx_labels', + txLabelMaps: 'tx_labels_map', + certificateFields: 'certificate_fields', + syncStates: 'sync_states' + } as Record + )[table] ?? table + const direct = await walletSnapshotSourceQuery(k, table, userId).select(name + '.*') + const indexed = await walletSnapshotSourceQuery(k, table, userId, 'v2', 'v2', 'v2', 'v2').select(name + '.*') + const order = (rows: unknown[]) => rows.map(row => JSON.stringify(row)).sort() + expect(order(indexed)).toEqual(order(direct)) + } + const ordinary = await pages(await source.openWalletReadSnapshot(identities[index])) + const archive = await pages(await openKnexSnapshotArchiveSource(source, identities[index])) + expect(Object.fromEntries(Object.entries(ordinary).map(([name, rows]) => [name, rows.length]))).toEqual( + Object.fromEntries(Object.entries(archive).map(([name, rows]) => [name, rows.length])) + ) + } + // A secondary-unique conflict with no child rows may replace a different PK. + const original = await k('sync_states').where('syncStateId', 1).first() + await replace(k, 'sync_states', { ...original, syncStateId: 101 }) + expect(await k('sync_states').where('syncStateId', 1)).toEqual([]) + await exact(rebuilt) + } finally { + await source.destroy() + await rm(directory, { recursive: true, force: true }) + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteAllFamilies.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteAllFamilies.test.ts new file mode 100644 index 000000000..87439c9a7 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteAllFamilies.test.ts @@ -0,0 +1,88 @@ +import fc from 'fast-check' +import { fixture, exact, value, keyOf, replace, tables } from '../../../test/utils/snapshotSqliteFixtures' + +test.each([false, true])( + 'all families retain exact memberships through replacements and nested ownership, recursive=%s', + async recursive => { + const k = await fixture('BINARY', recursive) + try { + for (const table of tables) for (const id of [1, 2]) await k(table).insert(value(table, id, id, id)) + await exact(k) + await k.raw( + 'CREATE TRIGGER nested_transaction_owner AFTER INSERT ON transactions WHEN NEW.userId=2 BEGIN UPDATE transactions SET userId=3 WHERE transactionId=NEW.transactionId; END' + ) + await replace(k, 'transactions', value('transactions', 1, 2, 2)) + await exact(k) + for (const table of tables) { + await replace(k, table, value(table, 1, 1, 3)) + await exact(k) + const query = k(table) + .where(keyOf(table, 2, 2)) + .update(value(table, 1, 1, 3)) + .toSQL() + await k.raw(query.sql.replace(/^update/i, 'UPDATE OR IGNORE'), query.bindings) + await exact(k) + } + } finally { + await k.destroy() + } + } +) + +test.each(['BINARY', 'NOCASE', 'RTRIM'])( + 'all-family independent source oracle survives %s conflict schedules', + async collation => { + const k = await fixture(collation, false) + try { + await fc.assert( + fc.asyncProperty( + fc.boolean(), + fc.array( + fc.record({ + table: fc.integer({ min: 0, max: 12 }), + kind: fc.integer({ min: 0, max: 4 }), + id: fc.integer({ min: 1, max: 6 }), + other: fc.integer({ min: 1, max: 6 }), + user: fc.integer({ min: 1, max: 3 }) + }), + { minLength: 1, maxLength: 12 } + ), + async (recursive, operations) => { + await k.raw('PRAGMA recursive_triggers=' + Number(recursive)) + for (const table of [...tables].reverse()) await k(table).delete() + await exact(k) + for (const op of operations) { + const table = tables[op.table] + if (op.kind === 0) await replace(k, table, value(table, op.id, op.other, op.user)) + else if (op.kind === 1) + await k(table) + .where(keyOf(table, op.id, op.other)) + .delete() + else if (op.kind < 4) { + const query = k(table) + .where(keyOf(table, op.id, op.other)) + .update(value(table, op.other, op.id, op.user)) + .toSQL() + await k.raw( + query.sql.replace(/^update/i, op.kind === 2 ? 'UPDATE OR REPLACE' : 'UPDATE OR IGNORE'), + query.bindings + ) + } else + await k.transaction(async trx => { + await trx(table) + .where(keyOf(table, op.id, op.other)) + .delete() + await trx.rollback() + }) + await exact(k) + } + } + ), + { numRuns: 300, seed: 3242026 } + ) + } finally { + await k.destroy() + } + }, + 30000 +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteBootstrap.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteBootstrap.test.ts new file mode 100644 index 000000000..da1d4c920 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteBootstrap.test.ts @@ -0,0 +1,254 @@ +import type { Knex } from 'knex' +import { fixture, value } from '../../../test/utils/snapshotSqliteFixtures' +import { installGeneration, names, progress, metadata } from '../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage, valid, type Position } from '../schema/snapshotSqliteIndexBootstrap' +import { readGenerationIndexState, migration } from '../schema/snapshotSqliteIndexState' + +const initial: Position = { stream: 0, afterId: 0, afterSecond: 0, afterText: '', complete: 0 } +test.each([ + { stream: -1 }, + { stream: 12 }, + { stream: 0.5 }, + { stream: NaN }, + { stream: Infinity }, + { afterId: -1 }, + { afterId: 0.5 }, + { afterId: 9007199254740992 }, + { afterSecond: 1 }, + { stream: 8, afterId: 1 }, + { stream: 9, afterSecond: 1 }, + { stream: 10, afterText: 'a' }, + { stream: 10, afterId: 1, afterText: 100 }, + { stream: 10, afterId: 1, afterText: 'a'.repeat(101) }, + { complete: 2 } +])('persisted position refuses unsupported state %j', patch => { + expect(valid({ ...initial, ...patch } as Position)).toBe(false) +}) +test('position accepts the last stream, safe ID boundary, exact 100-codepoint 400-byte field and both relation components', () => { + for (const state of [ + { ...initial, stream: 11, afterId: Number.MAX_SAFE_INTEGER, complete: 1 }, + { ...initial, stream: 10, afterId: 1, afterText: '😀'.repeat(100) }, + { ...initial, stream: 8, afterId: 1, afterSecond: 2 }, + { ...initial, stream: 9, afterId: 2, afterSecond: 1 } + ]) + expect(valid(state)).toBe(true) +}) + +async function prepareStream(k: Knex, stream: number) { + const plan = await installGeneration(k) + await k(progress).update({ complete: 1 }) + await k(progress).where('stream', stream).update({ complete: 0 }) + return plan +} +const invalidSources: Array<{ table: string; stream: number; patch: Record; message: string }> = [ + ...[0, -1, 9007199254740992].map(transactionId => ({ + table: 'transactions', + stream: 0, + patch: { transactionId }, + message: 'Invalid profile source identity' + })), + ...[0, 1.5, 'not an integer'].map(userId => ({ + table: 'transactions', + stream: 0, + patch: { userId }, + message: 'Invalid profile source identity' + })), + { table: 'tx_labels_map', stream: 8, patch: { txLabelId: 0 }, message: 'Invalid relation source identity' }, + { table: 'tx_labels_map', stream: 8, patch: { transactionId: 0 }, message: 'Invalid relation source identity' }, + { table: 'output_tags_map', stream: 9, patch: { outputTagId: -1 }, message: 'Invalid relation source identity' }, + { table: 'output_tags_map', stream: 9, patch: { outputId: 1.5 }, message: 'Invalid relation source identity' }, + { + table: 'certificate_fields', + stream: 10, + patch: { certificateId: 0 }, + message: 'Invalid certificate source identity' + }, + { table: 'certificate_fields', stream: 10, patch: { userId: 0 }, message: 'Invalid certificate source identity' }, + { + table: 'certificate_fields', + stream: 10, + patch: { fieldName: 'a'.repeat(101) }, + message: 'Invalid certificate source identity' + }, + { + table: 'certificate_fields', + stream: 10, + patch: { fieldName: '😀'.repeat(101) }, + message: 'Invalid certificate source identity' + }, + { + table: 'certificate_fields', + stream: 10, + patch: { fieldName: Buffer.from('field') }, + message: 'Invalid certificate source identity' + }, + { table: 'transactions', stream: 11, patch: { transactionId: 0 }, message: 'Invalid global source identity' }, + { table: 'transactions', stream: 11, patch: { userId: 0 }, message: 'Invalid global source identity' }, + { table: 'transactions', stream: 11, patch: { provenTxId: 0 }, message: 'Invalid global source identity' }, + { table: 'transactions', stream: 11, patch: { txid: 'a'.repeat(65) }, message: 'Invalid global source identity' }, + { table: 'transactions', stream: 11, patch: { txid: Buffer.from('txid') }, message: 'Invalid global source identity' } +] +test.each(invalidSources)( + 'historical $table identity refuses atomically at stream $stream: $patch', + async ({ table, stream, patch, message }) => { + const k = await fixture('BINARY', false, false) + try { + await k(table).insert({ ...value(table, 1, 1, 1), ...patch }) + const original = await k(table), + plan = await prepareStream(k, stream), + before = await k(progress).orderBy('stream') + await expect(copyGenerationPage(k, plan)).rejects.toThrow(message) + expect(await k(progress).orderBy('stream')).toEqual(before) + expect(await k(table)).toEqual(original) + for (const target of [names.profile, names.relation, names.certificate, names.edges]) + expect(await k(target)).toEqual([]) + } finally { + await k.destroy() + } + } +) +test.each([ + ['tx_labels_map', 'tx_labels', 8, 'Invalid relation source owner'], + ['output_tags_map', 'outputs', 9, 'Invalid relation source owner'], + ['certificate_fields', 'certificates', 10, 'Invalid certificate source owner'] +] as const)( + 'historical %s refuses invalid parent ownership and rolls back partial work', + async (table, parent, stream, message) => { + const k = await fixture('BINARY', false, false) + try { + await k(parent).insert(value(parent, 1, 1, 0)) + await k(table).insert(value(table, 1, 1, 1)) + const plan = await prepareStream(k, stream) + await expect(copyGenerationPage(k, plan)).rejects.toThrow(message) + expect(await k(progress).where('stream', stream).first()).toMatchObject({ ...initial, stream }) + expect(await k(stream === 10 ? names.certificate : names.relation)).toEqual([]) + } finally { + await k.destroy() + } + } +) +test.each([{ provenTxReqId: 0 }, { provenTxId: 0 }])( + 'historical global request identity %j refuses before adding edges', + async patch => { + const k = await fixture('BINARY', false, false) + try { + await k('transactions').insert({ ...value('transactions', 1, 1, 1), provenTxId: null }) + await k('proven_tx_reqs').insert({ ...value('proven_tx_reqs', 1, 1, 1), ...patch }) + const plan = await prepareStream(k, 11) + await expect(copyGenerationPage(k, plan)).rejects.toThrow('Invalid global request identity') + expect(await k(names.edges)).toEqual([]) + } finally { + await k.destroy() + } + } +) +test('asymmetric relation identities resolve each owner and retain the exact final tuple', async () => { + const k = await fixture('BINARY', false, false) + try { + await k('tx_labels').insert(value('tx_labels', 2, 2, 3)) + await k('transactions').insert(value('transactions', 7, 7, 5)) + await k('tx_labels_map').insert([ + { txLabelId: 2, transactionId: 8 }, + { txLabelId: 2, transactionId: 7 }, + { txLabelId: 1, transactionId: 7 } + ]) + const plan = await prepareStream(k, 8), + page = await copyGenerationPage(k, plan) + expect(page.copiedThrough).toEqual({ ...initial, stream: 8, afterId: 2, afterSecond: 8, complete: 1 }) + expect(await k(names.relation).orderBy(['snapshotLeftId', 'snapshotRightId', 'snapshotUserId'])).toEqual([ + { snapshotTableId: 0, snapshotUserId: 5, snapshotLeftId: 1, snapshotRightId: 7, snapshotMembership: 2 }, + { snapshotTableId: 0, snapshotUserId: 3, snapshotLeftId: 2, snapshotRightId: 7, snapshotMembership: 1 }, + { snapshotTableId: 0, snapshotUserId: 5, snapshotLeftId: 2, snapshotRightId: 7, snapshotMembership: 2 }, + { snapshotTableId: 0, snapshotUserId: 3, snapshotLeftId: 2, snapshotRightId: 8, snapshotMembership: 1 } + ]) + } finally { + await k.destroy() + } +}) +test('orphan fields keep their own profile and the maximum field spelling survives the cursor', async () => { + const k = await fixture('BINARY', false, false) + try { + await k('certificate_fields').insert([ + { ...value('certificate_fields', 1, 7, 3), fieldName: '😀'.repeat(100) }, + { ...value('certificate_fields', 1, 2, 5), fieldName: 'a' } + ]) + const plan = await prepareStream(k, 10), + page = await copyGenerationPage(k, plan) + expect(page.copiedThrough).toEqual({ ...initial, stream: 10, afterId: 7, afterText: '😀'.repeat(100), complete: 1 }) + expect(await k(names.certificate).orderBy('snapshotFieldName')).toEqual([ + { snapshotUserId: 5, snapshotCertificateId: 2, snapshotFieldName: 'a', snapshotMembership: 1 }, + { snapshotUserId: 3, snapshotCertificateId: 7, snapshotFieldName: '😀'.repeat(100), snapshotMembership: 1 } + ]) + } finally { + await k.destroy() + } +}) +test('unproven requests and null transaction IDs add only established global edges', async () => { + const k = await fixture('BINARY', false, false) + try { + await k('transactions').insert([ + { ...value('transactions', 1, 1, 1), txid: null, provenTxId: null }, + { ...value('transactions', 2, 2, 2), provenTxId: null } + ]) + await k('proven_tx_reqs').insert({ ...value('proven_tx_reqs', 5, 2, 1), provenTxId: null }) + const plan = await prepareStream(k, 11) + await copyGenerationPage(k, plan) + expect(await k(names.edges)).toEqual([{ transactionId: 2, requestId: 5, tableId: 0, rowId: 5, userId: 2 }]) + expect(await k(names.guards)).toEqual([]) + } finally { + await k.destroy() + } +}) +test.each(['missing', 'extra', 'wrong stream', 'second cursor', 'binary text'])( + 'corrupt %s progress refuses both copying and publication', + async kind => { + const k = await fixture('BINARY', false, false) + try { + const plan = await installGeneration(k) + if (kind === 'missing') await k(progress).where('stream', 11).delete() + if (kind === 'extra') await k(progress).insert({ ...initial, stream: 12 }) + if (kind === 'wrong stream') await k(progress).where('stream', 0).update({ stream: 12 }) + if (kind === 'second cursor') await k(progress).where('stream', 0).update({ afterSecond: 1 }) + if (kind === 'binary text') + await k(progress) + .where('stream', 10) + .update({ afterId: 1, afterText: Buffer.from('a') }) + const before = await k(progress).orderBy('stream') + await expect(copyGenerationPage(k, plan)).rejects.toThrow('Invalid generation progress') + await expect(readGenerationIndexState(k)).rejects.toThrow('Invalid generation progress') + expect(await k(progress).orderBy('stream')).toEqual(before) + } finally { + await k.destroy() + } + } +) +test('publication requires its configured schema journal and all completed streams', async () => { + const k = await fixture('BINARY', false, false) + try { + const plan = await installGeneration(k) + await k(progress).update({ complete: 1 }) + await copyGenerationPage(k, plan) + await k.raw("ATTACH DATABASE ':memory:' AS release_registry") + await k.schema.withSchema('release_registry').createTable('published', table => { + table.string('name').notNullable() + }) + const config = { schemaName: 'release_registry', tableName: 'published' } + await k.schema.createTable('published', table => { + table.string('name').notNullable() + }) + await k('published').insert({ name: migration }) + expect(await readGenerationIndexState(k, config)).toBe(false) + await k('published').withSchema('release_registry').insert({ name: migration }) + expect(await readGenerationIndexState(k, config)).toBe('v2') + await k(metadata).update({ complete: 0 }) + await expect(readGenerationIndexState(k, config)).rejects.toThrow('Published generation is incomplete') + await k(metadata).update({ complete: 1 }) + await k(progress).where('stream', 4).update({ complete: 0 }) + await expect(readGenerationIndexState(k, config)).rejects.toThrow('Completed generation has unfinished streams') + await k(progress).update({ complete: 1 }) + await k.schema.withSchema('release_registry').dropTable('published') + expect(await readGenerationIndexState(k, config)).toBe(false) + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteFileWal.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteFileWal.test.ts new file mode 100644 index 000000000..d749c65f1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteFileWal.test.ts @@ -0,0 +1,51 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { fixture, exact, value, replace, tables } from '../../../test/utils/snapshotSqliteFixtures' +import { installMembershipDraft } from '../../../test/utils/snapshotSqliteMaintenanceFixture' + +test.each([false, true])( + 'independent WAL writer and pinned reader survive trigger replacement, recursive=%s', + async recursive => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-identity-wal-')) + const filename = join(directory, 'fixture.sqlite') + const source = await fixture('BINARY', true, false, filename) + const writer = knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + let view: Knex.Transaction | undefined + try { + await writer.raw('PRAGMA recursive_triggers=' + Number(recursive)) + for (const table of tables) for (const id of [1, 2]) await source(table).insert(value(table, id, id, id)) + await exact(source) + view = await source.transaction() + const beforeRows = await view('transactions').orderBy('transactionId') + const beforeKeys = await view('snapshot_profile_keys').orderBy([ + 'snapshotTableId', + 'snapshotUserId', + 'snapshotRowId' + ]) + await installMembershipDraft(writer) + await replace(writer, 'transactions', value('transactions', 1, 2, 3)) + await exact(writer) + expect(await view('transactions').orderBy('transactionId')).toEqual(beforeRows) + expect( + await view('snapshot_profile_keys').orderBy(['snapshotTableId', 'snapshotUserId', 'snapshotRowId']) + ).toEqual(beforeKeys) + expect(await view('sqlite_master').where('name', 'snapshot_identity_transactions')).toEqual([]) + await view.commit() + view = undefined + expect(await source('transactions').orderBy('transactionId')).toEqual([value('transactions', 1, 2, 3)]) + await exact(source) + } finally { + if (view !== undefined) await view.rollback() + await writer.destroy() + await source.destroy() + await rm(directory, { recursive: true, force: true }) + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGeneration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGeneration.test.ts new file mode 100644 index 000000000..a6bcb67e6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGeneration.test.ts @@ -0,0 +1,383 @@ +import { readGenerationIndexState, migration } from '../schema/snapshotSqliteIndexState' +import { dropGenerationForDataDeletion } from '../schema/snapshotSqliteIndexMigration' +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import type { Knex } from 'knex' +import { fixture, exact, value, replace, tables } from '../../../test/utils/snapshotSqliteFixtures' +import { + installGeneration, + readPlan, + validateInstalled, + names, + metadata, + progress, + oldProgress +} from '../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage } from '../schema/snapshotSqliteIndexBootstrap' +import { legacyNames } from '../schema/snapshotSqliteMembership' +import { retiredTables } from '../schema/snapshotSqliteLegacyOwnership' +import { knex } from 'knex' + +function generation(k: Knex): Knex { + const mapping = new Map(Object.entries(legacyNames).map(([key, value]) => [value, names[key as keyof typeof names]])) + return new Proxy(k, { + apply(target, self, args: unknown[]) { + if (typeof args[0] === 'string' && mapping.has(args[0])) args[0] = mapping.get(args[0]) + return Reflect.apply(target, self, args) + } + }) +} +async function finish(k: Knex, plan: Awaited>) { + for (let page = 0; page < 100; page++) if ((await copyGenerationPage(k, plan)).complete) return + throw new Error('Rebuild did not finish') +} + +test.each(['BINARY', 'NOCASE', 'RTRIM'])( + 'fresh %s generation repairs stale indexes while low-key writes continue', + async collation => { + const k = await fixture(collation, false, false) + try { + for (const table of tables) for (const id of [1, 2]) await k(table).insert(value(table, id, id, id)) + await replace(k, 'transactions', value('transactions', 1, 2, 3)) + await expect(exact(k)).rejects.toThrow() + const source = await k('transactions') + const plan = await installGeneration(k) + expect(await k('transactions')).toEqual(source) + for (const table of oldProgress) expect((await k(table)).every(row => row.complete === 0)).toBe(true) + let pages = 0 + for (; pages < 100; pages++) { + const result = await copyGenerationPage(k, plan) + if (result.complete) break + await replace(k, 'transactions', value('transactions', 1, 2, (pages % 2) + 1)) + await replace(k, 'certificates', value('certificates', 1, 1, (pages % 2) + 1)) + await replace(k, 'proven_tx_reqs', value('proven_tx_reqs', 1, 2, (pages % 2) + 1)) + } + expect(pages).toBe(12) + await exact(generation(k)) + expect(await k(metadata).first('complete')).toEqual({ complete: 1 }) + expect(await installGeneration(k)).toEqual(plan) + await finish(k, plan) + await exact(generation(k)) + } finally { + await k.destroy() + } + } +) + +test('page rollback keeps its cursor and memberships together; resumed copies are bounded', async () => { + const k = await fixture('BINARY', false, false) + try { + for (let start = 0; start < 600; start += 100) + await k('transactions').insert( + Array.from({ length: 100 }, (_, i) => value('transactions', start + i + 1, start + i + 1, 1)) + ) + const plan = await installGeneration(k) + const failure = new Error('synthetic checkpoint write failure') + const inject = (query: { sql: string }) => { + if (query.sql.startsWith('update `' + progress + '`')) throw failure + } + k.on('query', inject) + try { + await expect(copyGenerationPage(k, plan)).rejects.toBe(failure) + } finally { + k.off('query', inject) + } + expect(await k(names.profile)).toEqual([]) + expect(await k(progress).where('stream', 0).first()).toMatchObject({ afterId: 0, complete: 0 }) + const first = await copyGenerationPage(k, plan) + expect(first.copiedThrough).toMatchObject({ afterId: 256, complete: 0 }) + expect(await k(names.profile)).toHaveLength(256) + await replace(k, 'transactions', value('transactions', 1, 2, 2)) + const resumed = await installGeneration(k) + expect((await copyGenerationPage(k, resumed)).copiedThrough).toMatchObject({ + afterId: 512, + complete: 0 + }) + expect((await copyGenerationPage(k, resumed)).copiedThrough).toMatchObject({ + afterId: 600, + complete: 1 + }) + await finish(k, resumed) + await exact(generation(k)) + } finally { + await k.destroy() + } +}) + +test.each([ + ['tx_labels_map', 'txLabelId INTEGER,transactionId INTEGER,UNIQUE(transactionId,txLabelId)'], + ['tx_labels_map', 'txLabelId INTEGER,transactionId INTEGER,UNIQUE(txLabelId)'], + ['tx_labels_map', 'txLabelId INTEGER,transactionId INTEGER,UNIQUE(txLabelId DESC,transactionId)'], + ['tx_labels_map', 'txLabelId INTEGER,transactionId INTEGER,UNIQUE(txLabelId COLLATE NOCASE,transactionId)'], + ['tx_labels_map', 'txLabelId INTEGER,transactionId INTEGER,UNIQUE(txLabelId,transactionId COLLATE RTRIM)'], + [ + 'certificate_fields', + 'userId INTEGER,fieldName VARCHAR(100),certificateId INTEGER,fieldValue TEXT,UNIQUE(fieldName,certificateId COLLATE NOCASE)' + ] +])('unsupported native composite comparison refuses before writing: %s / %s', async (table, columns) => { + const k = await fixture('BINARY', false, false) + try { + await k.schema.dropTable(table) + await k.raw(`CREATE TABLE ?? (${columns})`, [table]) + const before = await k('sqlite_master').orderBy(['type', 'name']) + await expect(installGeneration(k)).rejects.toThrow('Unsupported composite identity comparison') + expect(await k('sqlite_master').orderBy(['type', 'name'])).toEqual(before) + } finally { + await k.destroy() + } +}) + +test('an index whose comparison differs from the source order refuses a sorting scan', async () => { + const k = await fixture('BINARY', false, false) + try { + await k.schema.dropTable('certificate_fields') + await k.raw( + 'CREATE TABLE certificate_fields(userId INTEGER,fieldName VARCHAR(100),certificateId INTEGER,fieldValue TEXT,UNIQUE(fieldName COLLATE NOCASE,certificateId))' + ) + const query = 'SELECT fieldName,certificateId FROM certificate_fields ORDER BY fieldName,certificateId LIMIT 1' + expect( + (await k.raw('EXPLAIN QUERY PLAN ' + query)).some((step: { detail: string }) => + step.detail.includes('TEMP B-TREE') + ) + ).toBe(true) + await expect(installGeneration(k)).rejects.toThrow('Composite source order mismatch') + expect(await k.schema.hasTable(metadata)).toBe(false) + } finally { + await k.destroy() + } +}) + +test('the SQLite generation refuses a different database client before issuing queries', async () => { + const k = knex({ client: 'mysql2' }) + const query = jest.fn() + k.on('query', query) + try { + await expect(readPlan(k)).rejects.toThrow('SQLite rebuild requires SQLite') + expect(query).not.toHaveBeenCalled() + } finally { + await k.destroy() + } +}) + +test('validation binds the observed source schema and rejects malformed generated definitions', async () => { + const k = await fixture('BINARY', false, false) + try { + const plan = await installGeneration(k) + await k.raw('CREATE INDEX extra_source_index ON tx_labels(userId)') + await expect(validateInstalled(k, plan)).rejects.toThrow('Rebuild source schema changed') + await k.raw('DROP INDEX extra_source_index') + await expect(validateInstalled(k, { ...plan, ddl: ['SELECT 1'] })).rejects.toThrow( + 'Invalid generated schema definition' + ) + await expect(validateInstalled(k, plan)).resolves.toBeUndefined() + } finally { + await k.destroy() + } +}) + +test.each(['absent', 'extra', 'wrong id', 'invalid complete', 'binary legacy'])( + 'generation metadata must be one correctly typed source-bound row: %s', + async kind => { + const k = await fixture('BINARY', false, false) + try { + const plan = await installGeneration(k) + if (kind === 'absent') await k(metadata).delete() + if (kind === 'extra') await k(metadata).insert({ ...(await k(metadata).first()), id: 1 }) + if (kind === 'wrong id') await k(metadata).update({ id: 1 }) + if (kind === 'invalid complete') await k(metadata).update({ complete: 2 }) + if (kind === 'binary legacy') await k.raw('UPDATE ?? SET legacy = ?', [metadata, Buffer.from([0])]) + await expect(validateInstalled(k, plan)).rejects.toThrow('Rebuild source binding mismatch') + } finally { + await k.destroy() + } + } +) + +test.each(oldProgress)('an incomplete prior index refuses replacement: %s', async table => { + const k = await fixture('BINARY', false, false) + try { + await k(table).update({ complete: false }) + await k('knex_migrations') + .where('name', 'like', `% snapshot ${table.split('_')[1]} %`) + .delete() + const before = await k('sqlite_master').orderBy(['type', 'name']) + await expect(installGeneration(k)).rejects.toThrow('Prior snapshot migrations must be complete') + expect(await k('sqlite_master').orderBy(['type', 'name'])).toEqual(before) + } finally { + await k.destroy() + } +}) + +test.each(retiredTables)('foreign views of %s prevent retiring their data', async table => { + const k = await fixture('BINARY', false, false) + try { + await k.raw('CREATE VIEW foreign_snapshot_reader AS SELECT * FROM ??', [table.toUpperCase()]) + await expect(installGeneration(k)).rejects.toThrow('Unowned object references legacy auxiliary data') + expect(await k.schema.hasTable(metadata)).toBe(false) + expect(await k.schema.hasTable(table)).toBe(true) + } finally { + await k.destroy() + } +}) + +test('a view using a known trigger name is still an unowned reader of legacy data', async () => { + const k = await fixture('BINARY', false, false) + try { + await k.raw('CREATE VIEW snapshot_profile_0_insert AS SELECT * FROM snapshot_profile_keys') + expect(await k('sqlite_master').where('name', 'snapshot_profile_0_insert').orderBy('type').select('type')).toEqual([ + { type: 'trigger' }, + { type: 'view' } + ]) + await expect(installGeneration(k)).rejects.toThrow('Unowned object references legacy auxiliary data') + expect(await k.schema.hasTable(metadata)).toBe(false) + } finally { + await k.destroy() + } +}) + +test('a foreign auxiliary trigger prevents retirement even when its body reads no legacy data', async () => { + const k = await fixture('BINARY', false, false) + try { + await k.raw('CREATE TRIGGER application_auxiliary_hook AFTER INSERT ON snapshot_profile_keys BEGIN SELECT 1; END') + await expect(installGeneration(k)).rejects.toThrow('Unowned object references legacy auxiliary data') + expect(await k.schema.hasTable(metadata)).toBe(false) + } finally { + await k.destroy() + } +}) + +test('orphan generation objects refuse adoption without replacing their contents', async () => { + const k = await fixture('BINARY', false, false) + try { + await k.schema.createTable(names.profile, table => { + table.text('foreignValue') + }) + await k(names.profile).insert({ foreignValue: 'preserve' }) + await expect(installGeneration(k)).rejects.toThrow('Orphan rebuild schema refuses adoption') + expect(await k(names.profile)).toEqual([{ foreignValue: 'preserve' }]) + expect(await k.schema.hasTable(metadata)).toBe(false) + } finally { + await k.destroy() + } +}) + +test('reserved source trigger names refuse while independent application triggers are retained', async () => { + const k = await fixture('BINARY', false, false) + try { + await k.raw('CREATE TRIGGER snapshot_profile_unowned AFTER INSERT ON tx_labels BEGIN SELECT 1; END') + await expect(installGeneration(k)).rejects.toThrow('Unknown legacy source trigger') + await k.raw('DROP TRIGGER snapshot_profile_unowned') + await k.raw('CREATE TABLE application_events(id INTEGER)') + await k.raw( + 'CREATE TRIGGER app_snapshot_profile_notice AFTER INSERT ON tx_labels BEGIN INSERT INTO application_events VALUES(NEW.txLabelId); END' + ) + await installGeneration(k) + await k('tx_labels').insert(value('tx_labels', 1, 1, 1)) + expect(await k('application_events')).toEqual([{ id: 1 }]) + } finally { + await k.destroy() + } +}) + +test('the installed generation owns every required secondary range index', async () => { + const k = await fixture('BINARY', false, false) + try { + await installGeneration(k) + const required = { + snapshot_relation_right_v2: ['snapshotTableId', 'snapshotUserId', 'snapshotRightId', 'snapshotLeftId'], + snapshot_relation_map_v2: ['snapshotTableId', 'snapshotLeftId', 'snapshotRightId', 'snapshotUserId'], + snapshot_certificate_parent_v2: ['snapshotCertificateId', 'snapshotUserId', 'snapshotFieldName'], + snapshot_certificate_lookup_v2: ['snapshotFieldName', 'snapshotCertificateId', 'snapshotUserId'], + snapshot_global_page_v2: ['tableId', 'userId', 'present', 'rowId'], + snapshot_global_target_v2: ['tableId', 'rowId', 'userId'], + snapshot_global_request_v2: ['requestId', 'transactionId'] + } + for (const [name, columns] of Object.entries(required)) { + const parts: Array<{ key: number; name: string; desc: number }> = await k.raw('PRAGMA index_xinfo(??)', [name]) + expect(parts.filter(part => part.key === 1).map(part => ({ name: part.name, desc: part.desc }))).toEqual( + columns.map(name => ({ name, desc: 0 })) + ) + } + } finally { + await k.destroy() + } +}) + +test.each(['tx_labels_map', 'output_tags_map', 'certificate_fields'])( + 'unsupported additional unique constraint on %s refuses atomically', + async table => { + const k = await fixture('BINARY', false, false) + try { + const column = table === 'tx_labels_map' ? 'transactionId' : table === 'output_tags_map' ? 'outputId' : 'userId' + await k.raw('CREATE UNIQUE INDEX unexpected_identity ON ??(??)', [table, column]) + const before = await k('sqlite_master').orderBy('name') + await expect(installGeneration(k)).rejects.toThrow('Unsupported composite identity constraints') + expect(await k('sqlite_master').orderBy('name')).toEqual(before) + } finally { + await k.destroy() + } + } +) + +test('changed source schema and missing owned triggers refuse resume without adopting the mismatch', async () => { + const k = await fixture('BINARY', false, false) + try { + const plan = await installGeneration(k) + await k.raw('CREATE INDEX unexpected_source_index ON tx_labels(userId)') + await expect(installGeneration(k)).rejects.toThrow('Rebuild source binding mismatch') + await k.raw('DROP INDEX unexpected_source_index') + await k.raw('DROP TRIGGER snapshot_identity_before_transactions_INSERT') + await expect(installGeneration(k)).rejects.toThrow('Rebuild schema definition mismatch') + expect(await k(metadata).first('complete')).toEqual({ complete: 0 }) + expect(plan.source).not.toBe('') + } finally { + await k.destroy() + } +}) + +test.each([ + 'missing progress', + 'extra progress', + 'negative cursor', + 'wrong text cursor', + 'partial composite cursor', + 'invalid completion', + 'wrong source binding', + 'negative retirement', + 'fractional retirement', + 'excess retirement', + 'published pending', + 'completed with pending streams', + 'missing lock', + 'changed lock' +])('malformed generation state refuses adoption: %s', async kind => { + const k = await fixture('BINARY', false, false) + try { + await k('transactions').insert(value('transactions', 1, 1, 1)) + await installGeneration(k) + if (kind === 'missing progress') await k(progress).where('stream', 0).delete() + if (kind === 'extra progress') + await k(progress).insert({ stream: 12, afterId: 0, afterSecond: 0, afterText: '', complete: 0 }) + if (kind === 'negative cursor') await k(progress).where('stream', 0).update({ afterId: -1 }) + if (kind === 'wrong text cursor') await k(progress).where('stream', 0).update({ afterText: 'unexpected' }) + if (kind === 'partial composite cursor') await k(progress).where('stream', 8).update({ afterId: 1 }) + if (kind === 'invalid completion') await k(progress).where('stream', 0).update({ complete: 2 }) + if (kind === 'wrong source binding') await k(metadata).update({ source: 'changed' }) + if (kind === 'negative retirement') await k(metadata).update({ retireTable: -1 }) + if (kind === 'fractional retirement') await k(metadata).update({ retireTable: 0.5 }) + if (kind === 'excess retirement') await k(metadata).update({ retireTable: 7 }) + if (kind === 'published pending') + await k('knex_migrations').insert({ name: migration, batch: 99, migration_time: new Date() }) + if (kind === 'completed with pending streams') await k(metadata).update({ complete: 1 }) + if (kind === 'missing lock') await k.schema.dropTable('snapshot_index_install_lock_v2') + if (kind === 'changed lock') await k.raw('ALTER TABLE snapshot_index_install_lock_v2 ADD COLUMN extra INTEGER') + const before = await k('transactions') + await expect(readGenerationIndexState(k)).rejects.toBeInstanceOf(WERR_INVALID_OPERATION) + if (kind === 'missing lock' || kind === 'changed lock') { + await expect(dropGenerationForDataDeletion(k)).rejects.toBeInstanceOf(WERR_INVALID_OPERATION) + expect(await k.schema.hasTable(names.profile)).toBe(true) + } + expect(await k('transactions')).toEqual(before) + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationCost.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationCost.test.ts new file mode 100644 index 000000000..36693e27f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationCost.test.ts @@ -0,0 +1,208 @@ +import { mkdtemp, rm, stat } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { performance } from 'node:perf_hooks' +import { retireGenerationPage } from '../schema/snapshotSqliteIndexRetirement' +import { fixture, value, tables } from '../../../test/utils/snapshotSqliteFixtures' +import { installGeneration, names, metadata } from '../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage } from '../schema/snapshotSqliteIndexBootstrap' +import { walletSnapshotSourceQuery } from './KnexWalletReadSnapshot' +import type { WalletSnapshotTable } from './WalletReadSnapshot' + +const selections: Array<{ + table: WalletSnapshotTable + keys: string[] + after: (number | string)[] +}> = [ + { table: 'transactions', keys: ['snapshotRowId'], after: [4999] }, + { table: 'txLabelMaps', keys: ['snapshotLeftId', 'snapshotRightId'], after: [4999, 4999] }, + { table: 'outputTagMaps', keys: ['snapshotLeftId', 'snapshotRightId'], after: [4999, 4999] }, + { + table: 'certificateFields', + keys: ['snapshotFieldName', 'snapshotCertificateId'], + after: ['field-4999', 4999] + }, + { table: 'provenTxs', keys: ['rowId'], after: [4999] }, + { table: 'provenTxReqs', keys: ['rowId'], after: [4999] } +] + +test('all generation page shapes use indexed profile/key ranges without a temporary sort', async () => { + const k = await fixture('BINARY', false, false) + try { + await installGeneration(k) + for (const selection of selections) { + const query = walletSnapshotSourceQuery(k, selection.table, 1, 'v2', 'v2', 'v2', 'v2').select('*') + if (selection.keys.length === 1) void query.where(selection.keys[0], '>', selection.after[0]) + else void query.whereRaw('(??,??)>(?,?)', [...selection.keys, ...selection.after]) + void query.orderBy(selection.keys).limit(32) + const sql = query.toSQL() + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + sql.sql, sql.bindings) + expect(plan.some(step => step.detail.includes('SEARCH snapshot_') && step.detail.includes('_v2'))).toBe(true) + expect( + plan.filter(step => step.detail.includes('TEMP B-TREE') || step.detail.startsWith('SCAN snapshot_')) + ).toEqual([]) + } + } finally { + await k.destroy() + } +}) + +test('no-op and unrelated source updates do not amplify auxiliary writes', async () => { + const k = await fixture('BINARY', false, false) + try { + await k.schema.alterTable('transactions', table => table.text('description').notNullable().defaultTo('')) + const plan = await installGeneration(k) + await k('transactions').insert(value('transactions', 1, 1, 1)) + for (let i = 0; i < 30; i++) if ((await copyGenerationPage(k, plan)).complete) break + const total = async () => Number((await k.raw('SELECT total_changes() AS writes'))[0].writes) + const before = await total() + await k('transactions').where('transactionId', 1).update({ userId: 1, reference: 'r1', txid: 't1', provenTxId: 1 }) + expect((await total()) - before).toBe(1) + const same = await total() + await k('transactions').where('transactionId', 1).update({ description: 'updated payload' }) + expect((await total()) - same).toBe(1) + expect(await k(names.profile)).toHaveLength(1) + expect(await k(names.edges)).toHaveLength(1) + } finally { + await k.destroy() + } +}) + +test.each(['transactions', 'tx_labels_map', 'certificate_fields'])( + 'invalid leading %s identity cannot be skipped by the initial cursor', + async table => { + const k = await fixture('BINARY', false, false) + try { + const plan = await installGeneration(k) + const row = + table === 'transactions' + ? { ...value(table, 1, 1, 1), transactionId: -1 } + : table === 'tx_labels_map' + ? { txLabelId: -1, transactionId: 1 } + : { userId: 1, certificateId: -1, fieldName: '', fieldValue: 'v' } + await k(table).insert(row) + let rejected = false + for (let i = 0; i < 30; i++) { + try { + const result = await copyGenerationPage(k, plan) + if (result.complete) break + } catch (error) { + expect(error).toBeInstanceOf(Error) + rejected = true + break + } + } + expect(rejected).toBe(true) + expect(await k(metadata).first('complete')).toEqual({ complete: 0 }) + } finally { + await k.destroy() + } + } +) + +test('populated WAL measurement records bounded rebuild, indexed late pages and reusable retired space', async () => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-generation-measure-')) + const filename = join(directory, 'fixture.sqlite') + const k = await fixture('BINARY', false, false, filename) + const started = performance.now() + const footprint = async () => { + const base = await stat(filename) + const wal = await stat(filename + '-wal').catch(() => undefined) + return { + databaseBytes: base.size, + walBytes: wal?.size ?? 0, + pageCount: Number((await k.raw('PRAGMA page_count'))[0].page_count), + freePages: Number((await k.raw('PRAGMA freelist_count'))[0].freelist_count) + } + } + try { + for (const table of tables) + for (let start = 0; start < 1000; start += 100) { + const rows = Array.from({ length: 100 }, (_, index) => { + const id = start + index + 1 + return table === 'certificate_fields' + ? { userId: (id % 2) + 1, fieldName: 'a', certificateId: id, fieldValue: 'v' } + : value(table, id, id, (id % 2) + 1) + }) + await k(table).insert(rows) + } + const before = await footprint() + const copyStart = performance.now(), + plan = await installGeneration(k) + let peak = before.databaseBytes + before.walBytes, + copyPages = 0, + retirePages = 0, + maximumRetired = 0 + for (; copyPages < 100; copyPages++) { + const page = await copyGenerationPage(k, plan) + const now = await footprint() + peak = Math.max(peak, now.databaseBytes + now.walBytes) + if (page.complete) { + copyPages++ + break + } + } + expect(copyPages).toBe(49) + const copied = await footprint(), + copyMilliseconds = performance.now() - copyStart + const plans: Record = {} + for (const selected of selections) { + const after = selected.table === 'certificateFields' ? ['a', 900] : selected.keys.map(() => 900) + const query = walletSnapshotSourceQuery(k, selected.table, 1, 'v2', 'v2', 'v2', 'v2').select('*') + if (selected.keys.length === 1) void query.where(selected.keys[0], '>', after[0]) + else void query.whereRaw('(??,??)>(?,?)', [...selected.keys, ...after]) + void query.orderBy(selected.keys).limit(8) + const sql = query.toSQL(), + rows = await query + expect(rows).toHaveLength(8) + const steps: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + sql.sql, sql.bindings) + expect(steps.some(step => step.detail.includes('SEARCH snapshot_') && step.detail.includes('_v2'))).toBe(true) + expect( + steps.filter(step => step.detail.includes('TEMP B-TREE') || step.detail.startsWith('SCAN snapshot_')) + ).toEqual([]) + plans[selected.table] = steps + } + const retireStart = performance.now() + for (; retirePages < 200; retirePages++) { + const page = await retireGenerationPage(k, plan) + maximumRetired = Math.max(maximumRetired, page.removed) + expect(page.removed).toBeLessThanOrEqual(256) + const now = await footprint() + peak = Math.max(peak, now.databaseBytes + now.walBytes) + if (page.complete) { + retirePages++ + break + } + } + expect(retirePages).toBeLessThan(200) + const retired = await footprint() + expect(retired.freePages).toBeGreaterThan(copied.freePages) + const receipt = { + observedAt: new Date().toISOString(), + fixture: '1000 small rows in each of thirteen minimal standard source tables; two profiles', + production: false, + sourceRows: 13000, + sqliteVersion: (await k.raw('SELECT sqlite_version() AS version'))[0].version, + before, + copied, + retired, + peakObservedDatabaseAndWalBytes: peak, + copyPages, + retirePages, + maximumRetired, + copyMilliseconds, + retireMilliseconds: performance.now() - retireStart, + totalMilliseconds: performance.now() - started, + plans, + limitations: [ + 'fixed minimal records, not a complete large-wallet performance or resource-budget acceptance', + 'SQL range plans and returned-row counts do not measure internal VM row visits', + 'freed SQLite pages are reusable; database/WAL file shrink is not promised' + ] + } + console.log(JSON.stringify({ sqliteGenerationCost: receipt })) + } finally { + await k.destroy() + await rm(directory, { recursive: true, force: true }) + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationMigration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationMigration.test.ts new file mode 100644 index 000000000..a1bee1510 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationMigration.test.ts @@ -0,0 +1,56 @@ +import { fixture, value } from '../../../test/utils/snapshotSqliteFixtures' +import { migrateGeneration, refuseGenerationDowngrade } from '../schema/snapshotSqliteIndexMigration' +import { migration, readGenerationIndexState } from '../schema/snapshotSqliteIndexState' +import { names } from '../schema/snapshotSqliteIndexGeneration' +import { retiredTables } from '../schema/snapshotSqliteLegacyOwnership' + +test('registered-style migration yields between bounded pages and never auto-publishes its journal', async () => { + const k = await fixture('BINARY', false, false) + let timer: ReturnType | undefined + try { + for (let start = 0; start < 600; start += 100) + await k('transactions').insert( + Array.from({ length: 100 }, (_, i) => value('transactions', start + i + 1, start + i + 1, 1)) + ) + const source = await k('transactions').orderBy('transactionId') + let turns = 0, + active = true + const foreground = () => { + turns++ + if (active) timer = setTimeout(foreground, 0) + } + timer = setTimeout(foreground, 0) + try { + await migrateGeneration(k) + } finally { + active = false + if (timer) clearTimeout(timer) + } + expect(turns).toBeGreaterThan(2) + expect(await k('transactions').orderBy('transactionId')).toEqual(source) + expect(await k(names.profile)).toHaveLength(600) + for (const table of retiredTables) expect(await k.schema.hasTable(table)).toBe(false) + expect(await readGenerationIndexState(k)).toBe(false) + await k('knex_migrations').insert({ name: migration, batch: 99, migration_time: new Date() }) + expect(await readGenerationIndexState(k)).toBe('v2') + await migrateGeneration(k) + await expect(refuseGenerationDowngrade(k)).rejects.toThrow('downgrade is unsupported') + expect(await k('transactions').orderBy('transactionId')).toEqual(source) + } finally { + if (timer) clearTimeout(timer) + await k.destroy() + } +}) + +test('an outer SQLite transaction refuses before any schema edit', async () => { + const k = await fixture('BINARY', false, false) + try { + const before = await k('sqlite_master').orderBy('name') + await k.transaction(async trx => { + await expect(migrateGeneration(trx)).rejects.toThrow('independent bounded transactions') + }) + expect(await k('sqlite_master').orderBy('name')).toEqual(before) + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationReaders.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationReaders.test.ts new file mode 100644 index 000000000..ba7f36892 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationReaders.test.ts @@ -0,0 +1,142 @@ +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' +import { retireGenerationPage } from '../schema/snapshotSqliteIndexRetirement' +import { knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { installGeneration } from '../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage } from '../schema/snapshotSqliteIndexBootstrap' +import { readGenerationIndexState, migration } from '../schema/snapshotSqliteIndexState' +import { replace } from '../../../test/utils/snapshotSqliteFixtures' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { seedArchiveClosure } from '../../../test/utils/snapshotArchiveFixtures' +import { snapshotArchiveTables } from './archive/SnapshotArchive' +import { openKnexSnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' +import { readSnapshotProfileIndexState } from '../schema/snapshotProfileIndexMigration' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import type { SnapshotArchiveSource } from './archive/KnexSnapshotArchiveSource' + +async function pages(view: WalletReadSnapshot | SnapshotArchiveSource) { + const result: Record = {} + try { + if ('validateClosure' in view) await view.validateClosure() + for (const table of snapshotArchiveTables) { + let cursor: WalletSnapshotCursor | undefined + const rows: unknown[] = [] + for (let n = 0; n < 30; n++) { + const page = await view.readPage(table, cursor, { maxRows: 1, maxBytes: 131072 }) + rows.push(...page.rows) + if (page.done) { + result[table] = rows + break + } + cursor = page.cursor + } + expect(result[table]).toBeDefined() + } + return result + } finally { + await view.close() + } +} + +test.each(['ordinary', 'archive'])( + '%s pins legacy, pending and completed generations across independent WAL writes', + async kind => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-generation-reader-')) + const options = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + } + const k = knex(options), + writer = knex(options) + const legacySource = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex(options) + }) + let oldest: WalletReadSnapshot | SnapshotArchiveSource | undefined + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + const identity = '02' + '11'.repeat(32) + let pinned: WalletReadSnapshot | SnapshotArchiveSource | undefined + const open = async () => + kind === 'ordinary' + ? await source.openWalletReadSnapshot(identity) + : await openKnexSnapshotArchiveSource(source, identity) + try { + await k.raw('PRAGMA journal_mode=WAL') + await migrateBeforeSqliteGeneration(source, 'generation-reader', 'synthetic-generation-reader') + await source.makeAvailable() + const user = (await source.findOrInsertUser(identity)).user.userId + const other = (await source.findOrInsertUser('03' + '22'.repeat(32))).user.userId + await seedArchiveClosure(source, user, other) + const baseline = await pages(await open()) + pinned = await open() + await legacySource.makeAvailable() + oldest = + kind === 'ordinary' + ? await legacySource.openWalletReadSnapshot(identity) + : await openKnexSnapshotArchiveSource(legacySource, identity) + const plan = await installGeneration(writer) + const original = await writer('sync_states').where('syncStateId', 1).first() + await replace(writer, 'sync_states', { ...original, syncStateId: 101 }) + expect(await pages(pinned)).toEqual(baseline) + pinned = undefined + await expect(readSnapshotProfileIndexState(writer)).rejects.toThrow('incomplete') + expect(await readGenerationIndexState(writer)).toBe(false) + const pending = await pages(await open()) + expect(pending.syncStates.map(row => (row as { syncStateId: number }).syncStateId)).toEqual([3, 101]) + pinned = await open() + for (let i = 0; i < 50; i++) if ((await copyGenerationPage(writer, plan)).complete) break + // Complete local copy is not published as an indexed source before journaling. + expect(await readGenerationIndexState(writer)).toBe(false) + await writer('knex_migrations').insert({ + name: migration, + batch: 99, + migration_time: new Date() + }) + expect(await readGenerationIndexState(writer)).toBe('v2') + let retired = false + for (let i = 0; i < 100; i++) + if ((await retireGenerationPage(writer, plan)).complete) { + retired = true + break + } + expect(retired).toBe(true) + expect(await pages(oldest!)).toEqual(baseline) + oldest = undefined + await writer('sync_states').insert({ + ...original, + syncStateId: 102, + refNum: 'new-independent-state' + }) + expect(await pages(pinned)).toEqual(pending) + pinned = undefined + const queries: string[] = [] + const listen = (query: { sql: string }) => { + if (query.sql.includes('cross join') && query.sql.includes('_v2')) queries.push(query.sql) + } + k.on('query', listen) + try { + const readyView = await open() + if ('sourceSchema' in readyView) expect(readyView.sourceSchema).toBe(migration) + const ready = await pages(readyView) + expect(ready.syncStates.map(row => (row as { syncStateId: number }).syncStateId)).toEqual([3, 101, 102]) + expect(queries.length).toBeGreaterThan(0) + for (const table of snapshotArchiveTables.filter(table => table !== 'syncStates')) + expect(ready[table]).toEqual(baseline[table]) + } finally { + k.off('query', listen) + } + } finally { + if (oldest) await oldest.close() + if (pinned) await pinned.close() + await legacySource.destroy() + await writer.destroy() + await source.destroy() + await rm(directory, { recursive: true, force: true }) + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationRegistry.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationRegistry.test.ts new file mode 100644 index 000000000..899d01674 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationRegistry.test.ts @@ -0,0 +1,128 @@ +import { knex, type Knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { KnexMigrations, SNAPSHOT_SQLITE_INDEX_MIGRATION } from '../schema/KnexMigrations' +import { installGeneration, metadata, progress, names } from '../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage } from '../schema/snapshotSqliteIndexBootstrap' +import { readGenerationIndexState } from '../schema/snapshotSqliteIndexState' +import { + migrateGeneration, + refuseGenerationDowngrade, + dropGenerationForDataDeletion +} from '../schema/snapshotSqliteIndexMigration' +import { retiredTables } from '../schema/snapshotSqliteLegacyOwnership' +import { migrateBeforeSqliteGeneration } from '../../../test/utils/snapshotHistoricalMigrations' + +function fixture(tableName = 'knex_migrations') { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + migrations: { tableName } + }) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + return { k, source } +} +async function seed(source: StorageKnex, count: number) { + await source.makeAvailable() + const userId = (await source.findOrInsertUser('02' + '11'.repeat(32))).user.userId + for (let start = 0; start < count; start += 100) { + await source.knex('tx_labels').insert( + Array.from({ length: Math.min(100, count - start) }, (_, i) => ({ + userId, + label: `generation-${start + i}`, + isDeleted: false, + created_at: new Date('2026-01-01'), + updated_at: new Date('2026-01-01') + })) + ) + } +} + +test.each(['knex_migrations', 'custom_snapshot_journal'])( + 'registered migration resumes and publishes %s only after bounded copy/retirement', + async tableName => { + const { k, source } = fixture(tableName) + try { + await migrateBeforeSqliteGeneration(source, 'registry', 'synthetic-registry') + await seed(source, 600) + const before = await k('tx_labels').orderBy('txLabelId') + let pages = 0 + const failure = new Error('interrupt second generation source page') + const stop = (q: { sql: string }) => { + if (q.sql.startsWith('select `txLabelId` as `rowId`') && ++pages === 2) throw failure + } + k.on('query', stop) + try { + await expect(source.migrate('registry', 'synthetic-registry')).rejects.toBe(failure) + } finally { + k.off('query', stop) + } + expect(pages).toBe(2) + expect(await k(progress).where('stream', 3).first('afterId')).toEqual({ afterId: 256 }) + expect(await k(tableName).where('name', SNAPSHOT_SQLITE_INDEX_MIGRATION)).toEqual([]) + expect(await readGenerationIndexState(k, { tableName })).toBe(false) + expect((await k.raw('PRAGMA foreign_keys'))[0].foreign_keys).toBe(1) + await expect(source.migrate('registry', 'synthetic-registry')).resolves.toBe(SNAPSHOT_SQLITE_INDEX_MIGRATION) + expect(await readGenerationIndexState(k, { tableName })).toBe('v2') + expect(await k('tx_labels').orderBy('txLabelId')).toEqual(before) + expect(await k(names.profile).where('snapshotTableId', 3)).toHaveLength(600) + for (const table of retiredTables) expect(await k.schema.hasTable(table)).toBe(false) + expect(await k(tableName).where('name', SNAPSHOT_SQLITE_INDEX_MIGRATION)).toHaveLength(1) + await expect(source.migrate('registry', 'synthetic-registry')).resolves.toBe(SNAPSHOT_SQLITE_INDEX_MIGRATION) + const migrationSource = new KnexMigrations('test', 'registry', 'synthetic-registry', 1024) + expect(migrationSource.migrations[SNAPSHOT_SQLITE_INDEX_MIGRATION].config).toEqual({ transaction: false }) + await expect(k.migrate.down({ migrationSource, name: SNAPSHOT_SQLITE_INDEX_MIGRATION })).rejects.toThrow( + 'downgrade is unsupported' + ) + expect(await k('tx_labels').orderBy('txLabelId')).toEqual(before) + expect(await readGenerationIndexState(k, { tableName })).toBe('v2') + } finally { + await source.destroy() + } + } +) + +test.each(['complete', 'unpublished'] as const)( + 'explicit dropAllData preserves its contract for a %s generation', + async phase => { + const { k, source } = fixture() + try { + await migrateBeforeSqliteGeneration(source, 'deletion', 'synthetic-deletion') + await seed(source, 3) + if (phase === 'complete') await source.migrate('deletion', 'synthetic-deletion') + else await copyGenerationPage(k, await installGeneration(k)) + expect(await k.schema.hasTable(metadata)).toBe(true) + await source.dropAllData() + expect(await k.schema.hasTable(metadata)).toBe(false) + expect(await k.schema.hasTable('snapshot_index_install_lock_v2')).toBe(false) + expect(await k.schema.hasTable('users')).toBe(false) + expect(await k.schema.hasTable('tx_labels')).toBe(false) + expect(await k('knex_migrations')).toEqual([]) + expect(await k('sqlite_master').where('name', 'like', 'snapshot_%')).toEqual([]) + await source.dropAllData() + } finally { + await source.destroy() + } + } +) + +test('MySQL retains its existing migration and deletion behavior without SQLite operations', async () => { + const database = { client: { config: { client: 'mysql2' } } } as Knex + await expect(migrateGeneration(database)).resolves.toBeUndefined() + await expect(refuseGenerationDowngrade(database)).resolves.toBeUndefined() + await expect(dropGenerationForDataDeletion(database)).resolves.toBeUndefined() +}) + +test('empty migration adapters without a dialect label do not issue SQLite cleanup commands', async () => { + const currentVersion = jest.fn(async (_config: Knex.MigratorConfig) => 'none') + const down = jest.fn(), + raw = jest.fn() + const database = { client: { config: {} }, migrate: { currentVersion, down }, raw } as unknown as Knex + await StorageKnex.prototype.dropAllData.call({ knex: database } as StorageKnex) + expect(currentVersion).toHaveBeenCalledTimes(1) + expect(currentVersion.mock.calls[0][0].disableTransactions).toBe(false) + expect(down).not.toHaveBeenCalled() + expect(raw).not.toHaveBeenCalled() +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationRetirement.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationRetirement.test.ts new file mode 100644 index 000000000..2318cd432 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteGenerationRetirement.test.ts @@ -0,0 +1,245 @@ +import { WERR_INVALID_OPERATION } from '../../sdk/WERR_errors' +import { fixture, value } from '../../../test/utils/snapshotSqliteFixtures' +import { installGeneration, names, metadata } from '../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage } from '../schema/snapshotSqliteIndexBootstrap' +import { retireGenerationPage } from '../schema/snapshotSqliteIndexRetirement' +import { retiredTables } from '../schema/snapshotSqliteLegacyOwnership' + +test('obsolete generation cleanup is bounded, rollback-safe, resumable and preserves source data', async () => { + const k = await fixture('BINARY', false, false) + try { + for (let start = 0; start < 600; start += 100) + await k('transactions').insert( + Array.from({ length: 100 }, (_, i) => value('transactions', start + i + 1, start + i + 1, 1)) + ) + const source = await k('transactions').orderBy('transactionId') + const plan = await installGeneration(k) + await expect(retireGenerationPage(k, plan)).rejects.toThrow('before generation copy') + for (let i = 0; i < 100; i++) if ((await copyGenerationPage(k, plan)).complete) break + expect(await retireGenerationPage(k, plan)).toMatchObject({ + removed: 256, + complete: false, + table: retiredTables[0] + }) + expect(await k(retiredTables[0])).toHaveLength(344) + const failure = new Error('synthetic retirement rollback') + const inject = (_response: unknown, query: { sql: string }) => { + if (query.sql.startsWith('delete from `snapshot_global_edges`')) throw failure + } + k.on('query-response', inject) + try { + await expect(retireGenerationPage(k, plan)).rejects.toBe(failure) + } finally { + k.off('query-response', inject) + } + expect(await k(retiredTables[0])).toHaveLength(344) + const resumed = await installGeneration(k) + let completed = false + for (let i = 0; i < 100; i++) + if ((await retireGenerationPage(k, resumed)).complete) { + completed = true + break + } + expect(completed).toBe(true) + for (const table of retiredTables) expect(await k.schema.hasTable(table)).toBe(false) + expect(await k('transactions').orderBy('transactionId')).toEqual(source) + expect(await k(names.profile)).toHaveLength(600) + expect(await k(names.edges)).toHaveLength(600) + expect(await retireGenerationPage(k, resumed)).toEqual({ + complete: true, + removed: 0, + table: undefined + }) + } finally { + await k.destroy() + } +}) + +test('ownership drift refuses cleanup before deleting any legacy rows', async () => { + const k = await fixture('BINARY', false, false) + try { + await k('transactions').insert(value('transactions', 1, 1, 1)) + const plan = await installGeneration(k) + for (let i = 0; i < 100; i++) if ((await copyGenerationPage(k, plan)).complete) break + await k.raw( + 'CREATE TRIGGER foreign_old_edge AFTER DELETE ON snapshot_global_edges BEGIN DELETE FROM transactions; END' + ) + await expect(retireGenerationPage(k, plan)).rejects.toThrow('schema changed') + expect(await k('transactions')).toHaveLength(1) + expect(await k(metadata).first('retireTable')).toEqual({ retireTable: 0 }) + } finally { + await k.destroy() + } +}) + +test.each(['source-trigger', 'aux-trigger', 'reference-view'])( + 'unowned %s prevents replacement without changing schema', + async kind => { + const k = await fixture('BINARY', false, false) + try { + const sql = + kind === 'source-trigger' + ? 'CREATE TRIGGER snapshot_profile_custom AFTER INSERT ON transactions BEGIN SELECT 1; END' + : kind === 'aux-trigger' + ? 'CREATE TRIGGER foreign_owned_aux AFTER DELETE ON snapshot_global_edges BEGIN DELETE FROM transactions; END' + : 'CREATE VIEW foreign_aux_view AS SELECT * FROM snapshot_profile_keys' + await k.raw(sql) + const before = await k('sqlite_master').orderBy('name') + await expect(installGeneration(k)).rejects.toThrow( + /Unknown legacy source trigger|Unowned object references legacy auxiliary data|Unknown legacy auxiliary trigger/ + ) + expect(await k('sqlite_master').orderBy('name')).toEqual(before) + } finally { + await k.destroy() + } + } +) + +test('shared logical rowId values across key families still retire at most256 physical rows', async () => { + const k = await fixture('BINARY', false, false) + try { + for (let start = 0; start < 300; start += 100) { + await k('transactions').insert( + Array.from({ length: 100 }, (_, i) => value('transactions', start + i + 1, start + i + 1, 1)) + ) + await k('proven_tx_reqs').insert( + Array.from({ length: 100 }, (_, i) => value('proven_tx_reqs', start + i + 1, start + i + 1, 1)) + ) + } + const plan = await installGeneration(k) + for (let i = 0; i < 100; i++) if ((await copyGenerationPage(k, plan)).complete) break + let before = 0, + removed = 0 + for (let i = 0; i < 100; i++) { + before = Number((await k('snapshot_global_keys').count({ count: '*' }).first())!.count) + const page = await retireGenerationPage(k, plan) + if (page.table === 'snapshot_global_keys') { + removed = page.removed + expect(removed).toBe(256) + const after = Number((await k('snapshot_global_keys').count({ count: '*' }).first())!.count) + expect(before - after).toBe(256) + break + } + } + expect(before).toBe(600) + expect(removed).toBe(256) + expect(await k('transactions')).toHaveLength(300) + expect(await k(names.keys)).toHaveLength(600) + } finally { + await k.destroy() + } +}) + +test.each(['{', 'null', '[null]', '[{"type":"table","name":"foreign","tbl_name":"users","sql":null}]'])( + 'invalid retirement ownership receipt refuses before deletion: %s', + async legacy => { + const k = await fixture('BINARY', false, false) + try { + await k('transactions').insert(value('transactions', 1, 1, 1)) + const plan = await installGeneration(k) + let done = false + for (let n = 0; n < 50 && !done; n++) done = (await copyGenerationPage(k, plan)).complete + expect(done).toBe(true) + await k(metadata).update({ legacy }) + const before = await k('snapshot_global_edges') + await expect(retireGenerationPage(k, plan)).rejects.toBeInstanceOf(WERR_INVALID_OPERATION) + expect(await k('snapshot_global_edges')).toEqual(before) + expect(await k(metadata).first('retireTable')).toEqual({ retireTable: 0 }) + expect(await k('transactions')).toHaveLength(1) + } finally { + await k.destroy() + } + } +) + +test.each([-1, 257, 0.5, NaN, '256', undefined])( + 'an invalid driver deletion count %s rolls back actual retired rows and progress', + async invalid => { + const k = await fixture('BINARY', false, false) + const client = Object.getPrototypeOf(k.client) as { processResponse: (...args: unknown[]) => unknown } + const original = client.processResponse + let response: ReturnType | undefined + try { + for (let start = 0; start < 600; start += 100) + await k('transactions').insert( + Array.from({ length: 100 }, (_, i) => value('transactions', start + i + 1, start + i + 1, 1)) + ) + const plan = await installGeneration(k) + let complete = false + for (let page = 0; page < 100 && !complete; page++) complete = (await copyGenerationPage(k, plan)).complete + expect(complete).toBe(true) + const before = await k('snapshot_global_edges').orderBy('transactionId') + expect(before).toHaveLength(600) + let injected = 0 + response = jest.spyOn(client, 'processResponse').mockImplementation(function (this: unknown, ...args: unknown[]) { + const result = original.apply(this, args) + const query = args[0] + if ( + typeof query === 'object' && + query !== null && + 'sql' in query && + typeof query.sql === 'string' && + query.sql.startsWith('delete from `snapshot_global_edges`') + ) { + expect(result).toBe(256) + injected++ + return invalid + } + return result + }) + await expect(retireGenerationPage(k, plan)).rejects.toThrow('Invalid retirement row count') + expect(injected).toBe(1) + response.mockRestore() + response = undefined + expect(await k('snapshot_global_edges').orderBy('transactionId')).toEqual(before) + expect(await k(metadata).first('retireTable')).toEqual({ retireTable: 0 }) + expect(await k('transactions')).toHaveLength(600) + } finally { + response?.mockRestore() + await k.destroy() + } + } +) + +test('the final nonempty legacy table reports completion only after its empty-table drop commits', async () => { + const k = await fixture('BINARY', false, false) + try { + await k('certificates').insert(value('certificates', 1, 1, 1)) + for (let start = 0; start < 257; start += 100) + await k('certificate_fields').insert( + Array.from({ length: Math.min(100, 257 - start) }, (_, i) => ({ + userId: 1, + certificateId: 1, + fieldName: 'field-' + (start + i), + fieldValue: 'v' + })) + ) + const plan = await installGeneration(k) + let complete = false + for (let n = 0; n < 100 && !complete; n++) complete = (await copyGenerationPage(k, plan)).complete + expect(complete).toBe(true) + const table = retiredTables.at(-1)! + let finalPage: Awaited> | undefined + for (let n = 0; n < 20; n++) { + const page = await retireGenerationPage(k, plan) + expect(page.complete).toBe(false) + if (page.table === table) { + finalPage = page + break + } + } + expect(finalPage).toEqual({ complete: false, removed: 256, table }) + expect(await k(table)).toHaveLength(1) + expect(await k(metadata).first('retireTable')).toEqual({ retireTable: retiredTables.length - 1 }) + expect(await retireGenerationPage(k, plan)).toEqual({ complete: false, removed: 1, table }) + expect(await k.schema.hasTable(table)).toBe(true) + expect(await k(table)).toEqual([]) + expect(await retireGenerationPage(k, plan)).toEqual({ complete: true, removed: 0, table }) + expect(await k(metadata).first('retireTable')).toEqual({ retireTable: retiredTables.length }) + for (const retired of retiredTables) expect(await k.schema.hasTable(retired)).toBe(false) + expect(await k('certificate_fields')).toHaveLength(257) + expect(await k(names.certificate)).toHaveLength(257) + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteIdentity.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteIdentity.test.ts new file mode 100644 index 000000000..fc9b28883 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSqliteIdentity.test.ts @@ -0,0 +1,495 @@ +import { copyIdentityPage } from '../../../test/utils/snapshotSqliteIdentityFixture' +import { knex, type Knex } from 'knex' +import fc from 'fast-check' +import { + readIdentity, + identityDDL, + observeIdentity, + displacedIdentity, + finishIdentity, + type SourceIdentity, + type IdentityDefinition +} from '../schema/snapshotSqliteIdentity' + +const sources: Array< + SourceIdentity & { + fields: string + unique: string[] + values: (id: number) => Record + } +> = [ + { + table: 'transactions', + key: 'transactionId', + owner: 'userId', + fields: 'reference varchar(64)', + unique: ['reference'], + values: id => ({ reference: 'r' + id }) + }, + { + table: 'proven_txs', + key: 'provenTxId', + fields: 'txid varchar(64)', + unique: ['txid'], + values: id => ({ txid: 't' + id }) + }, + { + table: 'proven_tx_reqs', + key: 'provenTxReqId', + fields: 'txid varchar(64)', + unique: ['txid'], + values: id => ({ txid: 't' + id }) + }, + { + table: 'outputs', + key: 'outputId', + owner: 'userId', + fields: 'transactionId integer, vout integer', + unique: ['transactionId,vout,userId'], + values: id => ({ transactionId: 1, vout: id }) + }, + { + table: 'certificates', + key: 'certificateId', + owner: 'userId', + fields: 'type varchar(100), certifier varchar(130), serialNumber varchar(100)', + unique: ['userId,type,certifier,serialNumber'], + values: id => ({ type: 'a', certifier: 'b', serialNumber: 's' + id }) + }, + { + table: 'tx_labels', + key: 'txLabelId', + owner: 'userId', + fields: 'label varchar(300)', + unique: ['label,userId'], + values: id => ({ label: 'l' + id }) + }, + { + table: 'output_tags', + key: 'outputTagId', + owner: 'userId', + fields: 'tag varchar(150)', + unique: ['tag,userId'], + values: id => ({ tag: 't' + id }) + }, + { + table: 'output_baskets', + key: 'basketId', + owner: 'userId', + fields: 'name varchar(300)', + unique: ['name,userId'], + values: id => ({ name: 'b' + id }) + }, + { + table: 'commissions', + key: 'commissionId', + owner: 'userId', + fields: 'transactionId integer', + unique: ['transactionId'], + values: id => ({ transactionId: id }) + }, + { + table: 'sync_states', + key: 'syncStateId', + owner: 'userId', + fields: 'refNum varchar(100)', + unique: ['refNum'], + values: id => ({ refNum: 'r' + id }) + } +] + +async function database(source: (typeof sources)[number], collation: string, recursive: boolean) { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + await k.raw('PRAGMA recursive_triggers=' + Number(recursive)) + const fields = source.fields.replaceAll(/varchar\(\d+\)/g, value => value + ' COLLATE ' + collation) + await k.raw( + `CREATE TABLE ${source.table}(${source.key} integer PRIMARY KEY${source.owner ? ',userId integer NOT NULL' : ''},${fields},${source.unique.map(value => 'UNIQUE(' + value + ')').join(',')})` + ) + const definition = await readIdentity(k, source) + for (const ddl of identityDDL(definition)) await k.raw(ddl) + await k.raw('CREATE TABLE observed(id integer,userId integer)') + return { k, definition } +} + +async function observers(k: Knex, d: IdentityDefinition) { + for (const event of ['INSERT', 'UPDATE', 'DELETE'] as const) { + if (event !== 'DELETE') { + await k.raw( + `CREATE TRIGGER witness_before_${event} BEFORE ${event} ON ${d.source.table} BEGIN ${observeIdentity(d, event === 'UPDATE')} END` + ) + } + const observed = + event === 'DELETE' + ? `INSERT INTO observed VALUES(OLD.${d.source.key},${d.source.owner ? 'OLD.' + d.source.owner : 'NULL'});` + : `INSERT INTO observed SELECT ${d.source.key},${d.source.owner ?? 'NULL'} FROM (${displacedIdentity(d, event === 'UPDATE')});` + await k.raw( + `CREATE TRIGGER witness_after_${event} AFTER ${event} ON ${d.source.table} BEGIN ${observed} ${finishIdentity(d, event)} END` + ) + } +} + +function row(source: (typeof sources)[number], id: number, owner = 1, unique = id) { + return { [source.key]: id, ...(source.owner ? { userId: owner } : {}), ...source.values(unique) } +} + +async function same(k: Knex, d: IdentityDefinition) { + const columns = d.columns.map(column => column.name) + expect(await k(d.table).select(columns).orderBy(d.source.key)).toEqual( + await k(d.source.table).select(columns).orderBy(d.source.key) + ) +} + +async function copy(k: Knex, d: IdentityDefinition) { + let after: number | undefined = 0 + do { + const current: number = after + after = await k.transaction(trx => copyIdentityPage(trx, d, current, 1)) + } while (after !== undefined) +} + +test.each(sources)('captures unbootstrapped primary and unique conflicts in $table', async source => { + for (const collation of ['BINARY', 'NOCASE', 'RTRIM']) + for (const recursive of [false, true]) { + const { k, definition: d } = await database(source, collation, recursive) + try { + await k(source.table).insert([row(source, 1), row(source, 2), row(source, 3)]) + await observers(k, d) + const replacement = row(source, 1, 1, 2) + const insert = k(source.table).insert(replacement).toSQL() + await k.raw(insert.sql.replace(/^insert/i, 'INSERT OR REPLACE'), insert.bindings) + expect(await k('observed').orderBy('id')).toEqual([1, 2].map(id => ({ id, userId: source.owner ? 1 : null }))) + await copy(k, d) + await same(k, d) + await k('observed').delete() + const update = k(source.table) + .where(source.key, 3) + .update({ [source.key]: 1 }) + .toSQL() + await k.raw(update.sql.replace(/^update/i, 'UPDATE OR IGNORE'), update.bindings) + expect(await k('observed')).toEqual([]) + await same(k, d) + } finally { + await k.destroy() + } + } +}) + +test.each(['BINARY', 'NOCASE', 'RTRIM'])( + 'identity schedules match source state under %s', + async collation => { + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 0, max: sources.length - 1 }), + fc.boolean(), + fc.array( + fc.record({ + kind: fc.integer({ min: 0, max: 5 }), + id: fc.integer({ min: 1, max: 6 }), + other: fc.integer({ min: 1, max: 6 }), + owner: fc.integer({ min: 1, max: 3 }) + }), + { minLength: 1, maxLength: 25 } + ), + async (index, recursive, operations) => { + const source = sources[index] + const { k, definition: d } = await database(source, collation, recursive) + try { + await observers(k, d) + for (const op of operations) { + const value = row(source, op.id, op.owner, op.other) + if (op.kind < 2) { + const query = k(source.table).insert(value).toSQL() + await k.raw( + query.sql.replace(/^insert/i, op.kind === 0 ? 'INSERT OR REPLACE' : 'INSERT OR IGNORE'), + query.bindings + ) + } else if (op.kind < 4) { + const query = k(source.table) + .where(source.key, op.id) + .update(row(source, op.other, op.owner, op.id)) + .toSQL() + await k.raw( + query.sql.replace(/^update/i, op.kind === 2 ? 'UPDATE OR REPLACE' : 'UPDATE OR IGNORE'), + query.bindings + ) + } else if (op.kind === 4) { + await k(source.table).where(source.key, op.id).delete() + } else { + await k.transaction(async trx => { + await trx(source.table).where(source.key, op.id).delete() + await trx.rollback() + }) + } + await same(k, d) + } + } finally { + await k.destroy() + } + } + ), + { numRuns: 300, seed: 3242026 } + ) + }, + 30000 +) + +test('nullable unique values do not identify unrelated rows', async () => { + const source = sources[0] + const { k, definition: d } = await database(source, 'BINARY', false) + try { + await observers(k, d) + await k(source.table).insert([ + { transactionId: 1, userId: 1, reference: null }, + { transactionId: 2, userId: 2, reference: null } + ]) + await same(k, d) + expect(await k('observed')).toEqual([]) + } finally { + await k.destroy() + } +}) + +test.each([ + ['BINARY', 'A', false], + ['NOCASE', 'A', true], + ['RTRIM', 'a ', true] +] as const)('unique comparison preserves exact spelling under %s', async (collation, spelling, displaced) => { + const { k, definition: d } = await database(sources[0], collation, false) + try { + await k('transactions').insert({ transactionId: 1, userId: 1, reference: 'a' }) + await observers(k, d) + await k.raw('INSERT OR REPLACE INTO transactions(transactionId,userId,reference) VALUES(2,2,?)', [spelling]) + expect(await k('observed')).toEqual(displaced ? [{ id: 1, userId: 1 }] : []) + await copy(k, d) + await same(k, d) + expect((await k(d.table).where('transactionId', 2).first()).reference).toBe(spelling) + } finally { + await k.destroy() + } +}) + +test('bootstrap refuses a caller without a transaction before copying rows', async () => { + const { k, definition: d } = await database(sources[0], 'BINARY', false) + try { + await k('transactions').insert(row(sources[0], 1)) + await expect(copyIdentityPage(k, d, 0)).rejects.toThrow('requires a transaction') + expect(await k(d.table)).toEqual([]) + } finally { + await k.destroy() + } +}) + +test.each([false, true])('nested ownership changes retain every displaced owner, recursive=%s', async recursive => { + for (const registerNestedFirst of [false, true]) { + const { k, definition: d } = await database(sources[0], 'BINARY', recursive) + try { + await k('transactions').insert(row(sources[0], 1, 1)) + const nested = () => + k.raw( + 'CREATE TRIGGER nested_owner AFTER INSERT ON transactions WHEN NEW.userId=2 BEGIN UPDATE transactions SET userId=3 WHERE transactionId=NEW.transactionId; END' + ) + if (registerNestedFirst) await nested() + await observers(k, d) + if (!registerNestedFirst) await nested() + await k.raw('INSERT OR REPLACE INTO transactions(transactionId,userId,reference) VALUES(1,2,?)', ['r1']) + expect(await k('observed')).toEqual( + expect.arrayContaining([ + { id: 1, userId: 1 }, + { id: 1, userId: 2 } + ]) + ) + await same(k, d) + expect((await k('transactions').first()).userId).toBe(3) + } finally { + await k.destroy() + } + } +}) + +function openIdentitySchema() { + return knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) +} + +test.each([ + 'id INTEGER,owner INTEGER,label VARCHAR(100)', + 'id INTEGER,owner INTEGER,label VARCHAR(100),PRIMARY KEY(id,owner)', + 'id INTEGER,owner INTEGER PRIMARY KEY,label VARCHAR(100)', + 'id BIGINT PRIMARY KEY,owner INTEGER,label VARCHAR(100)' +])('unsupported native numeric identity refuses before creating witnesses: %s', async columns => { + const k = openIdentitySchema() + try { + await k.raw(`CREATE TABLE candidate_identity(${columns})`) + await expect(readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' })).rejects.toThrow( + 'Unsupported numeric identity' + ) + expect(await k.schema.hasTable('snapshot_identity_candidate_identity')).toBe(false) + } finally { + await k.destroy() + } +}) + +test('partial unique constraints cannot establish complete conflict witnesses', async () => { + const k = openIdentitySchema() + try { + await k.raw('CREATE TABLE candidate_identity(id INTEGER PRIMARY KEY,owner INTEGER,label VARCHAR(100))') + await k.raw('CREATE UNIQUE INDEX conditional_identity ON candidate_identity(label) WHERE owner=1') + await expect(readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' })).rejects.toThrow( + 'Unsupported partial unique identity' + ) + } finally { + await k.destroy() + } +}) + +test('mixed expression and column unique keys refuse unsupported comparison semantics', async () => { + const k = openIdentitySchema() + try { + await k.raw('CREATE TABLE candidate_identity(id INTEGER PRIMARY KEY,owner INTEGER,label VARCHAR(100))') + await k.raw('CREATE UNIQUE INDEX expression_identity ON candidate_identity(owner,LOWER(label))') + await expect(readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' })).rejects.toThrow( + 'Unsupported unique identity comparison' + ) + } finally { + await k.destroy() + } +}) + +test.each([ + 'label TEXT', + 'label BLOB', + 'label REAL', + 'label VARCHAR(0)', + 'label UNSIGNED BIGINT', + 'label INTEGER UNSIGNED', + 'label VARCHAR(100) GENERATED ALWAYS AS (CAST(id AS TEXT)) VIRTUAL', + 'label VARCHAR(100) GENERATED ALWAYS AS (CAST(id AS TEXT)) STORED' +])('unsupported or generated unique column refuses witness adoption: %s', async column => { + const k = openIdentitySchema() + try { + await k.raw(`CREATE TABLE candidate_identity(id INTEGER PRIMARY KEY,owner INTEGER,${column},UNIQUE(label))`) + await expect(readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' })).rejects.toThrow( + 'Unsupported identity column' + ) + } finally { + await k.destroy() + } +}) + +test('a missing ownership column refuses metadata adoption with its stable error', async () => { + const k = openIdentitySchema() + try { + await k.raw('CREATE TABLE candidate_identity(id INTEGER PRIMARY KEY,label VARCHAR(100) UNIQUE)') + await expect(readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' })).rejects.toThrow( + 'Unsupported identity column' + ) + } finally { + await k.destroy() + } +}) + +test('witness primary components are explicitly non-nullable', async () => { + const k = openIdentitySchema() + try { + await k.raw('CREATE TABLE candidate_identity(id INTEGER PRIMARY KEY,owner INTEGER,label VARCHAR(100) UNIQUE)') + const identity = await readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' }) + for (const sql of identityDDL(identity)) await k.raw(sql) + const columns: Array<{ name: string; notnull: number }> = await k.raw('PRAGMA table_info(??)', [identity.table]) + expect(columns.map(column => ({ name: column.name, notnull: column.notnull }))).toEqual([ + { name: 'id', notnull: 1 }, + { name: 'owner', notnull: 1 }, + { name: 'label', notnull: 0 } + ]) + await expect(k(identity.table).insert({ id: 1, owner: null, label: 'a' })).rejects.toThrow('NOT NULL') + await expect(k(identity.table).insert({ id: null, owner: 1, label: 'a' })).rejects.toThrow('NOT NULL') + } finally { + await k.destroy() + } +}) + +test('quoted native identifiers retain exact ownership and conflict comparison', async () => { + const k = openIdentitySchema() + const table = 'records"archive', + key = 'numeric"id', + owner = 'owner"id', + label = 'exact"label' + try { + await k.raw('CREATE TABLE ??(?? INTEGER PRIMARY KEY,?? INTEGER,?? VARCHAR(100) COLLATE NOCASE UNIQUE)', [ + table, + key, + owner, + label + ]) + const identity = await readIdentity(k, { table, key, owner }) + for (const sql of identityDDL(identity)) await k.raw(sql) + await k.raw(`CREATE TRIGGER quoted_before BEFORE INSERT ON ?? BEGIN ${observeIdentity(identity, false)} END`, [ + table + ]) + await k.raw(`CREATE TRIGGER quoted_after AFTER INSERT ON ?? BEGIN ${finishIdentity(identity, 'INSERT')} END`, [ + table + ]) + await k(table).insert({ [key]: 1, [owner]: 1, [label]: 'A' }) + const query = k(table) + .insert({ [key]: 2, [owner]: 2, [label]: 'a' }) + .toSQL() + await k.raw(query.sql.replace(/^insert/i, 'INSERT OR REPLACE'), query.bindings) + expect(await k(identity.table)).toEqual([{ [key]: 2, [owner]: 2, [label]: 'a' }]) + expect(await k(table)).toEqual(await k(identity.table)) + } finally { + await k.destroy() + } +}) + +test.each(['no key parts', 'unknown collation'])( + 'unexpected driver index metadata refuses before adoption: %s', + async kind => { + const k = openIdentitySchema() + const client = Object.getPrototypeOf(k.client) as { processResponse: (...args: unknown[]) => unknown } + const original = client.processResponse + let response: ReturnType | undefined + try { + await k.raw( + 'CREATE TABLE candidate_identity(id INTEGER PRIMARY KEY,owner INTEGER,label VARCHAR(100),UNIQUE(owner,label))' + ) + let injected = 0 + response = jest.spyOn(client, 'processResponse').mockImplementation(function (this: unknown, ...args: unknown[]) { + const result = original.apply(this, args) + const query = args[0] + if ( + typeof query === 'object' && + query !== null && + 'sql' in query && + typeof query.sql === 'string' && + query.sql.startsWith('PRAGMA index_xinfo(') + ) { + const parts = result as Array<{ name: string | null; key: number; coll: string }> + expect(parts.filter(part => part.key !== 0).map(part => part.name)).toEqual(['owner', 'label']) + injected++ + return kind === 'no key parts' + ? parts.filter(part => part.key === 0) + : parts.map(part => (part.name === 'label' ? { ...part, coll: 'UNSUPPORTED_COLLATION' } : part)) + } + return result + }) + await expect(readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' })).rejects.toThrow( + 'Unsupported unique identity comparison' + ) + expect(injected).toBe(1) + response.mockRestore() + response = undefined + expect((await readIdentity(k, { table: 'candidate_identity', key: 'id', owner: 'owner' })).unique).toEqual([ + [ + { name: 'owner', collation: 'BINARY' }, + { name: 'label', collation: 'BINARY' } + ] + ]) + } finally { + response?.mockRestore() + await k.destroy() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.integration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.integration.test.ts new file mode 100644 index 000000000..7ee4b76ce --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.integration.test.ts @@ -0,0 +1,1631 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { spawn } from 'node:child_process' +import { once } from 'node:events' +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { WalletStorageManager } from '../WalletStorageManager' +import { snapshotSyncTables, type SnapshotSyncCheckpoint, type SnapshotSyncStorage } from './SnapshotSync' +import type { WalletReadSnapshot } from './WalletReadSnapshot' +import { runSnapshotSyncSession } from './runSnapshotSyncSession' +import { KnexSnapshotSyncDestination } from './KnexSnapshotSyncDestination' +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import { SnapshotCancelledError } from './SnapshotCancelledError' +import { runInSeries } from '../../utility/runInSeries' +import { WERR_UNAUTHORIZED } from '../../sdk/WERR_errors' + +const identity = '02' + '11'.repeat(32) +const foreignIdentity = '03' + '22'.repeat(32) +const when = new Date('2026-01-01T00:00:00.000Z') +const stores: StorageKnex[] = [] +const directories: string[] = [] +function deferred() { + let resolve!: () => void + const promise = new Promise(r => { + resolve = r + }) + return { promise, resolve } +} +const pendingOperations: Promise[] = [] +function observe(operation: Promise): Promise { + // Keep secondary cleanup failures attached when a preceding fault assertion + // fails. Every primary result is still awaited/asserted by its owning test. + void operation.catch(() => undefined) + pendingOperations.push(operation) + return operation +} +async function waitForBoundary(boundary: Promise, operation: Promise): Promise { + await Promise.race([ + boundary, + operation.then(() => { + throw new Error('Operation completed before the required concurrency boundary') + }) + ]) +} +async function fixture(count = 130, wal = true, disabled?: 'source' | 'destination') { + const directory = await mkdtemp(join(tmpdir(), 'wallet-snapshot-sync-')) + directories.push(directory) + async function open(name: string, snapshotSync = true) { + const store = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + snapshotSync, + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, `${name}.sqlite`) }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 1000 + }) + }) + stores.push(store) + if (wal) await store.knex.raw('PRAGMA journal_mode = WAL') + await store.migrate(name, name) + await store.makeAvailable() + const { user } = await store.findOrInsertUser(identity) + await store.updateUser(user.userId, { activeStorage: 'source' }) + return store + } + const source = await open('source', disabled !== 'source') + const destination = await open('destination', disabled !== 'destination') + const { user: foreign } = await source.findOrInsertUser(foreignIdentity) + const user = (await source.findUserByIdentityKey(identity))! + await runInSeries( + Array.from({ length: count }, (_, index) => index), + async index => { + await source.insertTxLabel({ + txLabelId: 0, + userId: user.userId, + created_at: when, + updated_at: when, + label: `label-${index}`, + isDeleted: index % 3 === 0 + }) + await source.insertTxLabel({ + txLabelId: 0, + userId: foreign.userId, + created_at: when, + updated_at: when, + label: `foreign-${index}`, + isDeleted: false + }) + } + ) + const manager = new WalletStorageManager(identity, source, [destination]) + await manager.makeAvailable() + return { source, destination, manager, user, open } +} +async function advance( + view: WalletReadSnapshot, + destination: SnapshotSyncStorage, + target: number +): Promise { + let checkpoint = await destination.begin(view, view.user.activeStorage) + while (checkpoint.tableIndex < target) { + const page = await view.readPage(snapshotSyncTables[checkpoint.tableIndex], checkpoint.cursor) + const next = (await (await destination.prepare(checkpoint, page))()).checkpoint + expect(next.sequence).toBe(checkpoint.sequence + 1) + expect(next.tableIndex).toBe(checkpoint.tableIndex + Number(page.done)) + checkpoint = next + } + return checkpoint +} +afterEach(async () => { + jest.restoreAllMocks() + await runInSeries(stores.splice(0), store => store.destroy()) + await Promise.allSettled(pendingOperations.splice(0)) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('ordinary backup yields during preparation while its coherent view excludes foreground changes', async () => { + const { source, destination, manager, user } = await fixture() + const legacy = jest.spyOn(destination, 'processSyncChunk') + const gate = deferred() + const preparing = deferred() + const capability = destination.getSnapshotSync()! + let held = false + jest.spyOn(destination, 'getSnapshotSync').mockReturnValue({ + ...capability, + prepare: async (checkpoint, page) => { + const apply = await capability.prepare(checkpoint, page) + if (checkpoint.tableIndex === 3 && !held) { + held = true + preparing.resolve() + await gate.promise + } + return apply + } + }) + const backup = observe(manager.updateBackups()) + try { + await waitForBoundary(preparing.promise, backup) + await manager.runAsWriter(async () => { + await source.findOrInsertTxLabel(user.userId, 'foreground-after-view') + }) + } finally { + gate.resolve() + } + expect(await backup).toContain('snapshot complete') + const copied = await destination.findTxLabels({ partial: {} }) + expect(copied).toHaveLength(130) + expect(copied.every(row => row.label.startsWith('label-'))).toBe(true) + expect(copied.filter(row => row.isDeleted)).toHaveLength(44) + expect(legacy).not.toHaveBeenCalled() + expect(await destination.knex('sync_states')).toHaveLength(0) + expect(await destination.knex('snapshot_sync_ids').where({ entity: 'txLabel' })).toHaveLength(130) + await manager.updateBackups() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(131) +}) + +test('lost acknowledgement resumes the durable cursor and rejects a concurrently prepared stale page', async () => { + const { source, destination } = await fixture(5) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + const before = await advance(view, writer, 3) + const page = await view.readPage('txLabels', before.cursor, { maxRows: 2 }) + const apply = await writer.prepare(before, page) + const stale = await writer.prepare(before, page) + await apply() + const recovered = (await writer.checkpoint(identity, 'source'))! + expect(recovered.sequence).toBe(before.sequence + 1) + expect(recovered.cursor).toEqual(page.cursor) + expect(await writer.begin(view, view.user.activeStorage)).toEqual(recovered) + await expect(stale()).rejects.toThrow('session changed') + await expect(apply()).rejects.toThrow('already consumed') + const rest = await runSnapshotSyncSession( + { view, destination: writer, activeStorage: view.user.activeStorage, commit: op => op() }, + { maxItems: 2 } + ) + expect(rest.snapshotCheckpoint?.done).toBe(true) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(5) + await view.close() +}) + +test('page failure rolls data, mappings and checkpoint back together', async () => { + const { source, destination } = await fixture(3) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + const before = await advance(view, writer, 3) + await destination.knex.raw( + "CREATE TRIGGER reject_checkpoint BEFORE UPDATE ON snapshot_sync_sessions BEGIN SELECT RAISE(ABORT, 'checkpoint disk failure'); END" + ) + await expect((await writer.prepare(before, await view.readPage('txLabels')))()).rejects.toThrow( + 'checkpoint disk failure' + ) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) + expect(await destination.knex('snapshot_sync_ids').where({ entity: 'txLabel' })).toHaveLength(0) + expect(await writer.checkpoint(identity, 'source')).toEqual(before) + await destination.knex.raw('DROP TRIGGER reject_checkpoint') + await ( + await writer.prepare(before, await view.readPage('txLabels')) + )() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(3) + await view.close() +}) + +test.each(['before-row', 'before-checkpoint', 'before-commit', 'after-commit'] as const)( + 'a process killed %s recovers the atomic page outcome from its durable database', + async boundary => { + const { source, destination } = await fixture(3) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + const before = await advance(view, writer, 3) + const page = await view.readPage('txLabels', before.cursor, { maxRows: 2 }) + const filename = (destination.knex.client.config.connection as { filename: string }).filename + // Use the built package in a separate process: SIGKILL bypasses transaction + // catch/finally handlers, exercising the database's actual crash recovery. + const child = spawn( + process.execPath, + [ + '-e', + ` + const { knex } = require('knex') + const { StorageKnex } = require('./out/src/storage/StorageKnex.js') + const input = JSON.parse(process.argv[1], (key, value) => + key === 'created_at' || key === 'updated_at' ? new Date(value) : value) + const storage = new StorageKnex({ ...StorageKnex.defaultOptions(), chain: 'test', + knex: knex({ client: 'better-sqlite3', connection: { filename: input.filename }, + useNullAsDefault: true, pool: { min: 1, max: 1 } }) }) + async function hold() { process.send('at-boundary'); await new Promise(() => {}) } + async function run() { + await storage.makeAvailable() + if (input.boundary === 'before-row') { + const insert = storage.insertTxLabel.bind(storage) + storage.insertTxLabel = async (...args) => { await hold(); return insert(...args) } + } + if (input.boundary === 'before-checkpoint') { + const find = storage.findUsers.bind(storage) + let reads = 0 + storage.findUsers = async (...args) => { + if (++reads === 2) await hold() + return find(...args) + } + } + if (input.boundary === 'before-commit') { + const transaction = storage.transaction.bind(storage) + storage.transaction = (work, token) => transaction(async trx => { + const result = await work(trx) + await hold() + return result + }, token) + } + await (await storage.getSnapshotSync().prepare(input.before, input.page))() + if (input.boundary === 'after-commit') await hold() + throw new Error('Did not reach the requested boundary') + } + run().catch(error => { process.send({ error: error.message }); process.exitCode = 1 }) + `, + JSON.stringify({ filename, boundary, before, page }) + ], + { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] } + ) + let diagnostic = '' + child.stderr!.on('data', data => { + diagnostic += String(data) + }) + try { + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error(`Child did not reach ${boundary}: ${diagnostic}`)), 10000) + child.once('message', message => { + clearTimeout(timer) + if (message === 'at-boundary') resolve() + else reject(new Error(JSON.stringify(message))) + }) + child.once('error', error => { + clearTimeout(timer) + reject(error) + }) + child.once('exit', () => { + clearTimeout(timer) + reject(new Error(`Child exited early: ${diagnostic}`)) + }) + }) + const exited = once(child, 'exit') + expect(child.kill('SIGKILL')).toBe(true) + expect(await exited).toEqual([null, 'SIGKILL']) + const committed = boundary === 'after-commit' + const recovered = (await writer.checkpoint(identity, 'source'))! + expect(recovered.sequence).toBe(before.sequence + Number(committed)) + expect(recovered.cursor).toEqual(committed ? page.cursor : before.cursor) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(committed ? 2 : 0) + expect(await destination.knex('snapshot_sync_ids').where({ entity: 'txLabel' })).toHaveLength(committed ? 2 : 0) + expect((await destination.knex.raw('PRAGMA integrity_check'))[0].integrity_check).toBe('ok') + expect(await destination.knex.raw('PRAGMA foreign_key_check')).toEqual([]) + await runSnapshotSyncSession( + { view, destination: writer, activeStorage: view.user.activeStorage, commit: op => op() }, + { maxItems: 2 } + ) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(3) + expect(await destination.knex('snapshot_sync_ids').where({ entity: 'txLabel' })).toHaveLength(3) + } finally { + if (child.exitCode === null && child.signalCode === null) { + const exited = once(child, 'exit') + child.kill('SIGKILL') + await exited + } + await view.close() + } + } +) + +test('source replacement fences old pages without replacing legacy checkpoint JSON', async () => { + const { source, destination } = await fixture(4) + await destination.findOrInsertSyncStateAuth( + { identityKey: identity, userId: (await destination.findUserByIdentityKey(identity))!.userId }, + 'source', + 'source' + ) + const legacyBytes = (await destination.knex('sync_states').first()).syncMap + const writer = destination.getSnapshotSync()! + const oldView = (await source.getSnapshotSync()!.openSource(identity))! + const old = await advance(oldView, writer, 3) + const stale = await writer.prepare(old, await oldView.readPage('txLabels')) + await oldView.close() + const fresh = (await source.getSnapshotSync()!.openSource(identity))! + const started = await writer.begin(fresh, fresh.user.activeStorage) + expect(started.sequence).toBe(0) + expect(started.sessionId).not.toBe(old.sessionId) + await expect(stale()).rejects.toThrow('session changed') + await runSnapshotSyncSession( + { view: fresh, destination: writer, activeStorage: fresh.user.activeStorage, commit: op => op() }, + {} + ) + expect((await destination.knex('sync_states').first()).syncMap).toBe(legacyBytes) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(4) + await fresh.close() +}) + +test('independent primary change rejects prepared data', async () => { + const { source, destination } = await fixture(1) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + const before = await advance(view, writer, 3) + const apply = await writer.prepare(before, await view.readPage('txLabels')) + await destination.knex('users').where({ identityKey: identity }).update({ activeStorage: 'another-primary' }) + await expect(apply()).rejects.toThrow('session changed') + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) + await view.close() +}) + +test('an independent away-and-back primary transition cannot revive an old session', async () => { + const { source, destination } = await fixture(1) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + const before = await advance(view, writer, 3) + const apply = await writer.prepare(before, await view.readPage('txLabels')) + await destination.knex('users').where({ identityKey: identity }).update({ activeStorage: 'another-primary' }) + await destination.knex('users').where({ identityKey: identity }).update({ activeStorage: 'source' }) + await expect(apply()).rejects.toThrow('session changed') + await expect(writer.begin(view, view.user.activeStorage)).rejects.toThrow('binding changed') + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) + await view.close() +}) + +test('cancellation during commit reports durable outcome and releases source capacity', async () => { + const { source, destination, manager } = await fixture(3) + const controller = new AbortController() + const capability = destination.getSnapshotSync()! + jest.spyOn(destination, 'getSnapshotSync').mockReturnValue({ + ...capability, + prepare: async (checkpoint, page) => { + const apply = await capability.prepare(checkpoint, page) + return async () => { + const result = await apply() + if (checkpoint.tableIndex === 3) controller.abort() + return result + } + } + }) + const result = await manager.syncToWriterResumable(await manager.getAuth(), destination, { + signal: controller.signal, + maxItems: 1 + }) + expect(result.status).toBe('cancelled') + expect(result.snapshotCheckpoint).toEqual(await capability.checkpoint(identity, 'source')) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(1) + const next = await source.getSnapshotSync()!.openSource(identity) + expect(next).toBeDefined() + await next!.close() +}) + +test('non-WAL SQLite preserves legacy serialized backup', async () => { + const { source, destination, manager } = await fixture(2, false) + expect(await source.getSnapshotSync()!.openSource(identity)).toBeUndefined() + const legacy = jest.spyOn(destination, 'processSyncChunk') + await manager.updateBackups() + expect(legacy).toHaveBeenCalled() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + expect((await manager.syncToWriterResumable(await manager.getAuth(), destination)).mode).toBe('exclusive') +}) + +test('a backup borrowing an existing exclusive owner keeps the legacy reentrancy contract', async () => { + const { source, destination, manager } = await fixture(2) + const capability = jest.spyOn(source, 'getSnapshotSync') + const legacy = jest.spyOn(destination, 'processSyncChunk') + const log = await manager.runAsSync(active => manager.updateBackups(active)) + expect(log).toContain('BACKUP CURRENT ACTIVE TO 1 STORES') + expect(capability).not.toHaveBeenCalled() + expect(legacy).toHaveBeenCalled() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) +}) + +test.each([false, true])('primary replacement fences a pending legacy backup (switch back: %s)', async switchBack => { + const { source, destination, manager } = await fixture(2, false) + const entered = deferred() + const release = deferred() + const capability = source.getSnapshotSync()! + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ + ...capability, + openSource: async (...args) => { + entered.resolve() + await release.promise + return await capability.openSource(...args) + } + }) + const copying = observe(manager.updateBackups()) + let merge: jest.SpyInstance | undefined + let read: jest.SpyInstance | undefined + try { + await waitForBoundary(entered.promise, copying) + await manager.setActive('destination') + if (switchBack) await manager.setActive('source') + merge = jest.spyOn(destination, 'processSyncChunk') + read = jest.spyOn(source, 'getSyncChunk') + } finally { + release.resolve() + } + await expect(copying).rejects.toThrow('primary generation changed') + expect(read).not.toHaveBeenCalled() + expect(merge).not.toHaveBeenCalled() + expect(manager.getActiveStore()).toBe(switchBack ? 'source' : 'destination') +}) + +test('cancellation before source admission leaves durable session state untouched', async () => { + const { source, destination, manager } = await fixture(1) + const capability = source.getSnapshotSync()! + const opening = jest.fn(capability.openSource) + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ ...capability, openSource: opening }) + const controller = new AbortController() + controller.abort() + expect( + await manager.syncToWriterResumable(await manager.getAuth(), destination, { signal: controller.signal }) + ).toEqual({ + status: 'cancelled', + mode: 'paged', + pages: 0, + inserts: 0, + updates: 0 + }) + expect(opening).not.toHaveBeenCalled() + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) +}) + +test('destroy during the SQLite WAL probe rejects opening before creating a private reader', async () => { + const { source } = await fixture(0) + const entered = deferred() + const release = deferred() + jest.spyOn(source.knex.client, 'raw').mockImplementationOnce(() => { + entered.resolve() + return release.promise.then(() => [{ journal_mode: 'wal' }]) as never + }) + const privateReader = jest.spyOn(StorageKnex.prototype, 'openWalletReadSnapshot') + const opening = observe(source.getSnapshotSync()!.openSource(identity)) + let destroying: Promise | undefined + try { + await waitForBoundary(entered.promise, opening) + destroying = observe(source.destroy()) + } finally { + release.resolve() + } + await expect(opening).rejects.toThrow('destruction') + await destroying + expect(privateReader).not.toHaveBeenCalled() +}) + +const primaryCopyModes = ['snapshot', 'non-WAL', 'disabled-source', 'disabled-destination', 'resource-limit'] as const +async function primaryCopyFixture(mode: (typeof primaryCopyModes)[number]) { + const result = await fixture( + 1, + mode !== 'non-WAL', + mode === 'disabled-source' ? 'source' : mode === 'disabled-destination' ? 'destination' : undefined + ) + if (mode === 'resource-limit') { + const capability = result.source.getSnapshotSync()! + jest.spyOn(result.source, 'getSnapshotSync').mockReturnValue({ + ...capability, + openSource: async () => { + throw new SnapshotResourceLimitError('Synthetic retention admission limit') + } + }) + } + // The manager deliberately retains its earlier primary while an independent + // writer records a newer source selection. Push must forward the stored row. + await result.source.updateUser(result.user.userId, { + activeStorage: 'new-primary', + updated_at: new Date('2030-01-01T00:00:00.000Z') + }) + expect(result.manager.getActiveUser().activeStorage).toBe('source') + return result +} + +describe.each(primaryCopyModes)('primary metadata through %s', mode => { + test.each(['push', 'resumable-push', 'backup', 'borrowed-push'] as const)( + '%s preserves the newer source selection', + async operation => { + const { source, destination, manager, user } = await primaryCopyFixture(mode) + const auth = await manager.getAuth() + if (operation === 'push') await manager.syncToWriter(auth, destination) + else if (operation === 'resumable-push') await manager.syncToWriterResumable(auth, destination) + else if (operation === 'backup') await manager.updateBackups() + else await manager.runAsSync(active => manager.syncToWriter(auth, destination, active)) + expect((await destination.findUserByIdentityKey(identity))!.activeStorage).toBe('new-primary') + expect((await source.findUserByIdentityKey(identity))!.activeStorage).toBe('new-primary') + expect(await source.findTxLabels({ partial: { userId: user.userId } })).toHaveLength(1) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(1) + } + ) + + test.each(['pull', 'resumable-pull'] as const)('%s retains the destination selection', async operation => { + const { source, destination } = await primaryCopyFixture(mode) + const user = (await destination.findUserByIdentityKey(identity))! + await destination.updateUser(user.userId, { activeStorage: 'destination' }) + const manager = new WalletStorageManager(identity, destination) + await manager.makeAvailable() + if (operation === 'pull') await manager.syncFromReader(identity, source) + else await manager.syncFromReaderResumable(identity, source) + expect((await destination.findUserByIdentityKey(identity))!.activeStorage).toBe('destination') + expect((await source.findUserByIdentityKey(identity))!.activeStorage).toBe('new-primary') + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(1) + }) +}) + +test.each(['provider', 'adapter'] as const)( + 'snapshot pull through a %s preserves the active destination and remaps into an occupied profile', + async kind => { + const { source, destination } = await fixture(4) + const { user: foreign } = await destination.findOrInsertUser(foreignIdentity) + await destination.findOrInsertTxLabel(foreign.userId, 'foreign-existing') + const user = (await destination.findUserByIdentityKey(identity))! + await destination.updateUser(user.userId, { activeStorage: 'destination' }) + const manager = new WalletStorageManager(identity, destination) + await manager.makeAvailable() + const reader = + kind === 'provider' + ? source + : { + makeAvailable: source.makeAvailable.bind(source), + getSyncChunk: source.getSyncChunk.bind(source), + getSnapshotSync: source.getSnapshotSync.bind(source) + } + const legacy = jest.spyOn(destination, 'processSyncChunk') + const result = await manager.syncFromReaderResumable(identity, reader, { maxItems: 2 }) + expect(legacy).not.toHaveBeenCalled() + expect(result.snapshotCheckpoint?.done).toBe(true) + expect((await destination.findUserByIdentityKey(identity))!.activeStorage).toBe('destination') + expect(await destination.findTxLabels({ partial: { userId: user.userId } })).toHaveLength(4) + expect(await destination.findTxLabels({ partial: { userId: foreign.userId } })).toHaveLength(1) + } +) + +test('typed cancellation during source reading awaits cleanup and returns the last durable checkpoint', async () => { + const { source, destination, manager } = await fixture(3) + const controller = new AbortController() + const sourceCapability = source.getSnapshotSync()! + const closing = deferred() + const release = deferred() + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ + ...sourceCapability, + fallbackOnResourceError: false, + openSource: async (key, options) => { + expect(options?.signal).toBe(controller.signal) + const view = (await sourceCapability.openSource(key, options))! + return { + ...view, + readPage: async (...args) => { + if (args[0] === 'txLabels') { + controller.abort() + throw new SnapshotCancelledError('synthetic cancelled read') + } + return await view.readPage(...args) + }, + close: async () => { + closing.resolve() + await release.promise + await view.close() + } + } + } + }) + const legacy = jest.spyOn(destination, 'processSyncChunk') + const pending = observe( + manager.syncToWriterResumable(await manager.getAuth(), destination, { signal: controller.signal }) + ) + let finished = false + void pending.then( + () => { + finished = true + }, + () => { + finished = true + } + ) + try { + await waitForBoundary(closing.promise, pending) + expect(finished).toBe(false) + release.resolve() + const result = await pending + expect(result).toMatchObject({ status: 'cancelled', pages: 3, mode: 'paged' }) + expect(result.snapshotCheckpoint).toEqual(await destination.getSnapshotSync()!.checkpoint(identity, 'source')) + expect(legacy).not.toHaveBeenCalled() + expect(Reflect.get(source, 'snapshotSyncSource')).toBeUndefined() + } finally { + release.resolve() + await pending.catch(() => undefined) + } +}) + +test.each(['resource', 'validation', 'cleanup'] as const)( + 'source policy propagates %s failure without selecting fallback or mistaking it for cancellation', + async failureKind => { + const { source, destination, manager } = await fixture(1) + const controller = new AbortController() + const capability = source.getSnapshotSync()! + const failure = + failureKind === 'validation' + ? new Error('synthetic invalid row') + : new SnapshotResourceLimitError('synthetic resource failure') + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ + ...capability, + fallbackOnResourceError: false, + openSource: async (...args) => { + const view = (await capability.openSource(...args))! + return { + ...view, + readPage: async () => { + controller.abort() + throw failureKind === 'cleanup' ? new Error('synthetic row before cleanup failure') : failure + }, + close: async () => { + await view.close() + if (failureKind === 'cleanup') throw failure + } + } + } + }) + const legacy = jest.spyOn(destination, 'processSyncChunk') + await expect( + manager.syncToWriterResumable(await manager.getAuth(), destination, { signal: controller.signal }) + ).rejects.toBe(failure) + expect(legacy).not.toHaveBeenCalled() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) + expect(Reflect.get(source, 'snapshotSyncSource')).toBeUndefined() + } +) + +async function seedClosure(source: StorageKnex, userId: number, otherId: number): Promise { + const date = when.toISOString() + const timestamp = { created_at: date, updated_at: date } + const k = source.knex + await k('output_baskets').del() + for (const id of [1, 2, 3]) { + await k('proven_txs').insert({ + ...timestamp, + provenTxId: id, + txid: String(id).repeat(64), + height: id, + index: 0, + merklePath: Buffer.from([id, 0, 255]), + rawTx: Buffer.from([id, 1, 255]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + await k('transactions').insert({ + ...timestamp, + transactionId: id, + userId: id === 2 ? otherId : userId, + provenTxId: id === 3 ? null : id, + status: 'completed', + reference: `tx-${id}`, + isOutgoing: true, + satoshis: 0, + description: `tx-${id}`, + txid: String(id).repeat(64), + rawTx: Buffer.from([id, 2, 255]), + inputBEEF: Buffer.from([id, 3, 255]) + }) + await k('proven_tx_reqs').insert({ + ...timestamp, + provenTxReqId: id, + provenTxId: id, + txid: String(id).repeat(64), + status: 'completed', + attempts: 0, + notified: true, + history: '{}', + notify: '{}', + rawTx: Buffer.from([id, 4, 255]), + wasBroadcast: true + }) + await k('output_baskets').insert({ + ...timestamp, + basketId: id, + userId: id === 2 ? otherId : userId, + name: `basket-${id}`, + isDeleted: id === 3 + }) + await k('outputs').insert({ + ...timestamp, + outputId: id, + userId: id === 2 ? otherId : userId, + transactionId: id, + basketId: id, + spendable: false, + change: true, + vout: 0, + satoshis: 1, + providedBy: 'you', + purpose: '', + type: 'P2PKH', + lockingScript: Buffer.from([id, 5, 255]) + }) + await k('commissions').insert({ + ...timestamp, + commissionId: id, + userId: id === 2 ? otherId : userId, + transactionId: id, + satoshis: 0, + keyOffset: 'offset', + isRedeemed: true, + lockingScript: Buffer.from([id, 6, 255]) + }) + await k('output_tags').insert({ + ...timestamp, + outputTagId: id, + userId: id === 2 ? otherId : userId, + tag: `tag-${id}`, + isDeleted: id === 3 + }) + await k('output_tags_map').insert({ ...timestamp, outputTagId: id, outputId: id, isDeleted: id === 3 }) + await k('tx_labels').insert({ + ...timestamp, + txLabelId: id, + userId: id === 2 ? otherId : userId, + label: `label-${id}`, + isDeleted: id === 3 + }) + await k('tx_labels_map').insert({ ...timestamp, txLabelId: id, transactionId: id, isDeleted: id === 3 }) + await k('certificates').insert({ + ...timestamp, + certificateId: id, + userId: id === 2 ? otherId : userId, + serialNumber: `serial-${id}`, + type: 'type', + certifier: identity, + subject: identity, + revocationOutpoint: 'a'.repeat(64) + '.0', + signature: 'signature', + isDeleted: id === 3 + }) + for (const fieldName of ['a', 'Z', 'é', '😀']) + await k('certificate_fields').insert({ + ...timestamp, + certificateId: id, + userId: id === 2 ? otherId : userId, + fieldName, + fieldValue: `value-${id}`, + masterKey: 'key' + }) + await k('sync_states').insert({ + ...timestamp, + syncStateId: id, + userId: id === 2 ? otherId : userId, + storageIdentityKey: `peer-${id}`, + storageName: `peer-${id}`, + status: 'unknown', + init: true, + refNum: `state-${id}`, + syncMap: '{}', + when: date + }) + } + // Composite positions must handle repeated first keys and preserve deleted mappings. + await k('output_tags_map').insert({ ...timestamp, outputTagId: 1, outputId: 3, isDeleted: true }) + await k('tx_labels_map').insert({ ...timestamp, txLabelId: 1, transactionId: 3, isDeleted: true }) +} + +test('all replica relations and packed blobs remap into an occupied destination, including request-only proofs', async () => { + const { source, destination, manager, user } = await fixture(0) + const foreign = (await source.findUserByIdentityKey(foreignIdentity))! + await seedClosure(source, user.userId, foreign.userId) + const { user: occupied } = await destination.findOrInsertUser(foreignIdentity) + await destination.insertTransaction({ + transactionId: 0, + userId: occupied.userId, + created_at: when, + updated_at: when, + status: 'completed', + reference: 'occupied', + isOutgoing: false, + satoshis: 0, + description: '', + txid: 'c'.repeat(64) + }) + await source + .knex('proven_tx_reqs') + .where({ provenTxReqId: 1 }) + .update({ notify: JSON.stringify({ transactionIds: [1, 2, 3] }) }) + const insertProof = jest.spyOn(destination, 'insertProvenTx') + await manager.updateBackups() + expect( + insertProof.mock.calls.every( + ([proof]) => proof.rawTx instanceof Uint8Array && proof.merklePath instanceof Uint8Array + ) + ).toBe(true) + const destinationUser = (await destination.findUserByIdentityKey(identity))! + const transactions = await destination.findTransactions({ partial: { userId: destinationUser.userId } }) + expect(transactions).toHaveLength(2) + expect(await destination.findProvenTxs({ partial: {} })).toHaveLength(2) + const outputs = await destination.findOutputs({ partial: { userId: destinationUser.userId } }) + expect(outputs).toHaveLength(2) + for (const output of outputs) { + expect(transactions.some(tx => tx.transactionId === output.transactionId)).toBe(true) + const basket = (await destination.findOutputBaskets({ partial: { basketId: output.basketId } }))[0] + expect(basket.userId).toBe(destinationUser.userId) + expect(basket.name).toBe(`basket-${output.lockingScript![0]}`) + expect(output.lockingScript).toEqual([output.lockingScript![0], 5, 255]) + } + expect(await destination.findCertificates({ partial: { userId: destinationUser.userId } })).toHaveLength(2) + expect(await destination.findCertificateFields({ partial: { userId: destinationUser.userId } })).toHaveLength(8) + expect(await destination.findCommissions({ partial: { userId: destinationUser.userId } })).toHaveLength(2) + const requests = await destination.findProvenTxReqs({ partial: {} }) + expect(requests).toHaveLength(2) + expect(JSON.parse(requests.find(row => row.txid === '1'.repeat(64))!.notify).transactionIds.sort()).toEqual( + transactions.map(tx => tx.transactionId).sort() + ) + expect(await destination.knex('sync_states')).toHaveLength(0) + const unchangedRequest = jest.spyOn(destination, 'updateProvenTxReq') + await manager.updateBackups() + expect(unchangedRequest).not.toHaveBeenCalled() + expect(await destination.findTransactions({ partial: { userId: destinationUser.userId } })).toHaveLength(2) + expect(await destination.findTransactions({ partial: { userId: occupied.userId } })).toHaveLength(1) +}) + +test.each(['batch', 'history', 'notify', 'updated_at'] as const)( + 'a changed proof-request %s persists once while an unchanged copy stays quiescent', + async field => { + const { source, destination, manager, user } = await fixture(0) + const foreign = (await source.findUserByIdentityKey(foreignIdentity))! + await seedClosure(source, user.userId, foreign.userId) + await manager.updateBackups() + const value = { + batch: 'batch-a', + history: JSON.stringify({ notes: [{ what: 'fixture', when: '2026-01-02T00:00:00.000Z' }] }), + notify: JSON.stringify({ transactionIds: [1, 2, 3] }), + updated_at: new Date('2026-01-02T00:00:00.000Z') + }[field] + // A replicated metadata change need not carry a newer timestamp. Exercise + // each merge reason independently instead of having updated_at mask it. + await source + .knex('proven_tx_reqs') + .where({ provenTxReqId: 1 }) + .update({ + [field]: value instanceof Date ? value.toISOString() : value + }) + const writes = jest.spyOn(destination, 'updateProvenTxReq') + expect(await manager.updateBackups()).toContain('0 inserts, 0 updates') + expect(writes).toHaveBeenCalledTimes(1) + const copied = (await destination.findProvenTxReqs({ partial: { txid: '1'.repeat(64) } }))[0] + if (field === 'notify') { + const transactions = await destination.findTransactions({ + partial: { userId: (await destination.findUserByIdentityKey(identity))!.userId } + }) + expect(JSON.parse(copied.notify).transactionIds).toEqual(transactions.map(row => row.transactionId).sort()) + expect(copied.notified).toBe(false) + } else expect(copied[field]).toEqual(value) + await destination.updateProvenTxReq(copied.provenTxReqId, { notified: true }) + writes.mockClear() + expect(await manager.updateBackups()).toContain('0 inserts, 0 updates') + expect(writes).not.toHaveBeenCalled() + expect((await destination.findProvenTxReqs({ partial: { provenTxReqId: copied.provenTxReqId } }))[0].notified).toBe( + true + ) + if (field === 'batch') { + await source.updateProvenTxReq(1, { batch: 'conflicting-batch' }) + await expect(manager.updateBackups()).rejects.toThrow('merge batch not equal') + expect((await destination.findProvenTxReqs({ partial: { provenTxReqId: copied.provenTxReqId } }))[0].batch).toBe( + 'batch-a' + ) + } + } +) + +test.each([false, true])('primary replacement fences a prepared snapshot page (switch back: %s)', async switchBack => { + const { destination, manager } = await fixture(3) + const gate = deferred() + const preparing = deferred() + const capability = destination.getSnapshotSync()! + const applyLatePage = jest.fn() + jest.spyOn(destination, 'getSnapshotSync').mockReturnValue({ + ...capability, + prepare: async (checkpoint, page) => { + const apply = await capability.prepare(checkpoint, page) + if (checkpoint.tableIndex === 3) { + applyLatePage.mockImplementation(apply) + preparing.resolve() + await gate.promise + return applyLatePage + } + return apply + } + }) + const copying = observe(manager.updateBackups()) + try { + await waitForBoundary(preparing.promise, copying) + await manager.setActive('destination') + if (switchBack) await manager.setActive('source') + } finally { + gate.resolve() + } + await expect(copying).rejects.toThrow('primary generation changed') + expect(applyLatePage).not.toHaveBeenCalled() + expect(manager.getActiveStore()).toBe(switchBack ? 'source' : 'destination') + expect((await destination.findUserByIdentityKey(identity))!.activeStorage).toBe(switchBack ? 'source' : 'destination') +}) + +test('cancellation while queued never begins the prepared destination page', async () => { + const { destination, manager } = await fixture(3) + const controller = new AbortController() + const entered = deferred() + const release = deferred() + let blocking: Promise | undefined + const copying = observe( + manager.syncToWriterResumable(await manager.getAuth(), destination, { + signal: controller.signal, + onProgress: progress => { + if (progress.state === 'committing' && progress.snapshotCheckpoint?.tableIndex === 3) { + blocking = observe( + manager.runAsWriter(async () => { + entered.resolve() + await release.promise + }) + ) + } + } + }) + ) + try { + await waitForBoundary(entered.promise, copying) + controller.abort() + } finally { + release.resolve() + } + await blocking + const result = await copying + expect(result.status).toBe('cancelled') + expect(result.snapshotCheckpoint?.tableIndex).toBe(3) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) +}) + +test('an uncommitted auxiliary migration keeps ordinary backup on the compatible legacy path', async () => { + const { destination, manager } = await fixture(2) + await destination.knex('knex_migrations').where({ name: '2026-09-30-001 add durable snapshot sync' }).del() + expect(await destination.getSnapshotSync()!.supportsDestination()).toBe(false) + const legacy = jest.spyOn(destination, 'processSyncChunk') + await manager.updateBackups() + expect(legacy).toHaveBeenCalled() + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) +}) + +test('large valid rows select serialized fallback after bounded commits without losing counts or data', async () => { + const { source, destination, manager, user } = await fixture(2) + const destinationUser = (await destination.findUserByIdentityKey(identity))! + await destination.insertTxLabel({ + txLabelId: 0, + userId: destinationUser.userId, + created_at: new Date('2025-01-01'), + updated_at: new Date('2025-01-01'), + label: 'label-0', + isDeleted: false + }) + const bytes = new Uint8Array(200000).fill(173) + await source.insertTransaction({ + transactionId: 0, + userId: user.userId, + created_at: when, + updated_at: when, + status: 'completed', + reference: 'large', + isOutgoing: false, + satoshis: 0, + description: '', + rawTx: bytes as unknown as number[] + }) + const legacy = jest.spyOn(destination, 'processSyncChunk') + const modes: string[] = [] + const progress: Array<{ mode: string; state: string; pages: number; inserts: number; updates: number }> = [] + const result = await manager.syncToWriterResumable(await manager.getAuth(), destination, { + onProgress: p => { + modes.push(p.mode) + progress.push({ ...p }) + } + }) + expect(result.status).toBe('completed') + expect(result.mode).toBe('exclusive') + expect(result.inserts).toBe(2) + expect(result.updates).toBe(1) + const paged = progress.filter(p => p.mode === 'paged').at(-1)! + const exclusive = progress.filter(p => p.mode === 'exclusive') + expect(paged).toMatchObject({ inserts: 1, updates: 1 }) + expect(exclusive[0]).toMatchObject({ pages: paged.pages, inserts: 1, updates: 1 }) + expect(exclusive.at(-1)).toMatchObject({ pages: result.pages, inserts: 2, updates: 1 }) + expect(result.pages).toBe(paged.pages + exclusive.filter(p => p.state === 'committed').length) + expect(modes).toContain('paged') + expect(modes).toContain('exclusive') + expect(legacy).toHaveBeenCalled() + const transactions = await destination.findTransactions({ partial: {} }) + expect(transactions).toHaveLength(1) + expect(new Uint8Array(transactions[0].rawTx!)).toEqual(bytes) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + const fresh = await source.getSnapshotSync()!.openSource(identity) + await fresh!.close() +}) + +test.each(['admission', 'commit'] as const)( + 'expiry during destination %s selects fallback after physical cleanup', + async boundary => { + const { source, destination, manager } = await fixture(2) + const reader = source.getSnapshotSync()! + let view!: WalletReadSnapshot + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ + ...reader, + openSource: async (...args) => { + view = (await reader.openSource(...args))! + return view + } + }) + const writer = destination.getSnapshotSync()! + let expired = false + const expire = async (operation: () => Promise): Promise => { + expired = true + const clock = jest.spyOn(Date, 'now').mockReturnValue(view.expiresAt) + try { + return await operation() + } finally { + clock.mockRestore() + } + } + jest.spyOn(destination, 'getSnapshotSync').mockReturnValue({ + ...writer, + begin: async (...args) => + boundary === 'admission' && !expired ? await expire(() => writer.begin(...args)) : await writer.begin(...args), + prepare: async (checkpoint, page) => { + const apply = await writer.prepare(checkpoint, page) + return async () => + boundary === 'commit' && !expired && checkpoint.tableIndex === 3 ? await expire(apply) : await apply() + } + }) + const merge = destination.processSyncChunk.bind(destination) + const legacy = jest.spyOn(destination, 'processSyncChunk').mockImplementation(async (...args) => { + expect(view.isOpen).toBe(false) + await expect(view.closed).resolves.toBeUndefined() + return await merge(...args) + }) + const result = await manager.syncToWriterResumable(await manager.getAuth(), destination) + expect(result).toMatchObject({ status: 'completed', mode: 'exclusive', inserts: 2 }) + expect(legacy).toHaveBeenCalled() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + const fresh = (await reader.openSource(identity))! + await fresh.close() + } +) + +test('profile or allocation corruption never selects the legacy fallback', async () => { + const { source, destination, manager } = await fixture(1) + const capability = destination.getSnapshotSync()! + const legacy = jest.spyOn(destination, 'processSyncChunk') + jest.spyOn(destination, 'getSnapshotSync').mockReturnValue({ + ...capability, + prepare: async (checkpoint, page) => { + if (checkpoint.tableIndex === 3) (page.rows[0] as { userId: number }).userId = 999 + return await capability.prepare(checkpoint, page) + } + }) + await expect(manager.updateBackups()).rejects.toThrow('another profile') + expect(legacy).not.toHaveBeenCalled() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) + const next = await source.getSnapshotSync()!.openSource(identity) + await next!.close() +}) + +test('rejects invalid source bindings before creating a destination session', async () => { + const { source, destination } = await fixture(1) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const capability = destination.getSnapshotSync()! + const invalid = [ + { ...view, version: 2 }, + { ...view, snapshotId: 'a'.repeat(63) }, + { ...view, snapshotId: 'g'.repeat(64) }, + { ...view, expiresAt: NaN }, + { ...view, expiresAt: Date.now() - 1 }, + { ...view, expiresAt: Date.now() + 36000000 }, + { ...view, user: { ...view.user, userId: 0 } }, + { ...view, user: { ...view.user, userId: 1.5 } }, + { ...view, user: { ...view.user, identityKey: 'not-an-identity' } }, + { ...view, sourceStorage: { ...view.sourceStorage, storageIdentityKey: '' } }, + { ...view, sourceStorage: { ...view.sourceStorage, storageIdentityKey: 'a'.repeat(131) } }, + { ...view, sourceStorage: { ...view.sourceStorage, storageIdentityKey: 'destination' } }, + { ...view, sourceStorage: { ...view.sourceStorage, chain: 'main' } } + ] + for (const header of invalid) + await expect(capability.begin(header as typeof view, view.user.activeStorage)).rejects.toThrow() + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + await view.close() +}) + +test('rejects changed checkpoints and malformed pages without advancing durable progress', async () => { + const { source, destination } = await fixture(3) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const capability = destination.getSnapshotSync()! + const before = await advance(view, capability, 3) + const page = await view.readPage('txLabels', undefined, { maxRows: 1 }) + const changed = [ + { ...before, version: 2 }, + { ...before, sessionId: 'g'.repeat(64) }, + { ...before, sessionId: 'f'.repeat(64) }, + { ...before, snapshotId: 'f'.repeat(64) }, + { ...before, sequence: -1 }, + { ...before, sequence: 1.5 }, + { ...before, sequence: before.sequence + 1 }, + { ...before, tableIndex: -1 }, + { ...before, tableIndex: 0.5 }, + { ...before, tableIndex: 12, done: true }, + { ...before, destinationStorageIdentityKey: 'different' }, + { ...before, sourceStorageIdentityKey: 'different' }, + { ...before, identityKey: foreignIdentity } + ] + for (const checkpoint of changed) + await expect((async () => (await capability.prepare(checkpoint as typeof before, page))())()).rejects.toThrow() + const badPages = [ + { ...page, rows: [] }, + { ...page, rows: Array(1001).fill(page.rows[0]) }, + { ...page, done: undefined }, + { ...page, payloadBytes: -1 }, + { ...page, payloadBytes: Infinity }, + { ...page, payloadBytes: 16777217 }, + { ...page, payloadBytes: 1 }, + { ...page, cursor: undefined }, + { ...page, cursor: { ...page.cursor!, version: 2 } }, + { ...page, cursor: { ...page.cursor!, table: 'outputs' } }, + { ...page, cursor: { ...page.cursor!, after: [] } }, + { ...page, cursor: { ...page.cursor!, after: [999] } }, + { ...page, rows: [{ ...page.rows[0], userId: 999 }] }, + { ...page, rows: [{ ...page.rows[0], userId: undefined }] }, + { ...page, rows: [{ ...page.rows[0], txLabelId: 0 }], cursor: { ...page.cursor!, after: [0] } }, + { ...page, rows: [{ ...page.rows[0], txLabelId: NaN }] }, + { ...page, rows: [{ ...page.rows[0], label: { nested: true } }] }, + { ...page, rows: [{ ...page.rows[0], created_at: new Date(NaN) }] } + ] + for (const invalid of badPages) + await expect((async () => (await capability.prepare(before, invalid as typeof page))())()).rejects.toThrow() + expect(await capability.checkpoint(identity, 'source')).toEqual(before) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) + await view.close() +}) + +test('archive positions persist with rows, detach before proof I/O and reject an omitted durable binding', async () => { + const { source, destination } = await fixture(3) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + try { + let checkpoint = await advance(view, writer, 3) + const first = await view.readPage('txLabels', undefined, { maxRows: 1 }) + const second = await view.readPage('txLabels', first.cursor, { maxRows: 1 }) + const last = await view.readPage('txLabels', second.cursor, { maxRows: 1 }) + Reflect.set(first.cursor!, 'archivePosition', { version: 1, archiveId: 'c'.repeat(64), sequence: 3, rowOffset: 1 }) + checkpoint = (await (await writer.prepare(checkpoint, first))()).checkpoint + expect((await writer.checkpoint(identity, 'source'))!.cursor!.archivePosition).toEqual( + first.cursor!.archivePosition + ) + Reflect.set(second.cursor!, 'archivePosition', { ...first.cursor!.archivePosition, rowOffset: 2 }) + const preparing = writer.prepare(checkpoint, second) + Reflect.set(checkpoint.cursor!.archivePosition!, 'rowOffset', 999) + Reflect.set(second.cursor!.archivePosition!, 'rowOffset', 999) + checkpoint = (await (await preparing)()).checkpoint + expect(checkpoint.cursor!.archivePosition!.rowOffset).toBe(2) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + const missing = { ...checkpoint, cursor: { ...checkpoint.cursor!, archivePosition: undefined } } + const apply = await writer.prepare(missing, last) + await expect(apply()).rejects.toThrow('Snapshot session changed') + expect(await writer.checkpoint(identity, 'source')).toEqual(checkpoint) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + } finally { + await view.close() + } +}) + +test('source admission, expiry and destruction retain bounded reader ownership', async () => { + const { source, destination } = await fixture(0) + const capability = source.getSnapshotSync()! + for (const lifetimeMs of [0, -1, 1.5, Infinity, 3600001]) + await expect(capability.openSource(identity, { lifetimeMs })).rejects.toThrow('lifetimeMs') + const controller = new AbortController() + controller.abort() + await expect(capability.openSource(identity, { signal: controller.signal })).rejects.toThrow('cancelled') + await expect(capability.openSource('02' + '33'.repeat(32))).rejects.toThrow('existing wallet profile') + const view = (await capability.openSource(identity, { lifetimeMs: 10000 }))! + await expect(capability.openSource(identity)).rejects.toThrow('already has') + expect(await destination.getSnapshotSync()!.checkpoint(identity, 'missing')).toBeUndefined() + expect(await destination.getSnapshotSync()!.checkpoint(foreignIdentity, 'source')).toBeUndefined() + const clock = jest.spyOn(Date, 'now').mockReturnValue(view.expiresAt) + await expect(view.readPage('txLabels')).rejects.toThrow('expired') + await expect(view.closed).resolves.toBeUndefined() + clock.mockRestore() + const fresh = (await capability.openSource(identity))! + await source.destroy() + await fresh.closed + expect(fresh.isOpen).toBe(false) + await expect(capability.openSource(identity)).rejects.toThrow('destruction') +}) + +test('all push and pull APIs reject another profile before touching the peer', async () => { + const { manager, destination } = await fixture(0) + const peer = jest.spyOn(destination, 'makeAvailable') + await expect(manager.syncToWriter({ identityKey: foreignIdentity }, destination)).rejects.toThrow() + await expect(manager.syncToWriterResumable({ identityKey: foreignIdentity }, destination)).rejects.toThrow() + await expect(manager.syncFromReader(foreignIdentity, destination)).rejects.toBeInstanceOf(WERR_UNAUTHORIZED) + await expect(manager.syncFromReaderResumable(foreignIdentity, destination)).rejects.toBeInstanceOf(WERR_UNAUTHORIZED) + expect(peer).not.toHaveBeenCalled() +}) + +test.each(['source', 'destination'] as const)( + 'disabling snapshot sync on the %s preserves ordinary backup through the legacy path', + async disabled => { + const { source, destination, manager } = await fixture(2, true, disabled) + const store = disabled === 'source' ? source : destination + const legacy = jest.spyOn(destination, 'processSyncChunk') + expect(store.getSnapshotSync()).toBeUndefined() + expect(await store.knex.schema.hasTable('snapshot_sync_sessions')).toBe(true) + expect(store.supportsWalletReadSnapshot()).toBe(true) + expect(await manager.updateBackups()).toContain('serialized complete') + expect(legacy).toHaveBeenCalled() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + } +) + +test('profile selection and its update count are atomic with the first page', async () => { + const { source, destination, manager } = await fixture(0) + await destination + .knex('users') + .where({ identityKey: identity }) + .update({ activeStorage: 'old-primary', updated_at: '2000-01-01T00:00:00.000Z' }) + const controller = new AbortController() + const cancelled = await manager.syncToWriterResumable(await manager.getAuth(), destination, { + signal: controller.signal, + onProgress: progress => { + if (progress.state === 'reading') controller.abort() + } + }) + expect(cancelled.pages).toBe(0) + expect(cancelled.status).toBe('cancelled') + expect((await destination.findUserByIdentityKey(identity))!.activeStorage).toBe('old-primary') + const result = await manager.syncToWriterResumable(await manager.getAuth(), destination) + expect(result.status).toBe('completed') + expect(result.updates).toBe(1) + expect((await destination.findUserByIdentityKey(identity))!.activeStorage).toBe('source') + expect((await source.findUserByIdentityKey(identity))!.activeStorage).toBe('source') +}) + +test('ordinary copies retain their previous byte ceiling while a fresh manager initializes lazily', async () => { + const { source, destination } = await fixture(1) + const limits: number[] = [] + for (const store of [source, destination]) { + const capability = store.getSnapshotSync()! + jest.spyOn(store, 'getSnapshotSync').mockReturnValue({ + ...capability, + openSource: async (...args) => { + const view = (await capability.openSource(...args))! + return { + ...view, + get isOpen() { + return view.isOpen + }, + readPage: async (table, cursor, bounds) => { + limits.push(bounds!.maxBytes!) + return await view.readPage(table, cursor, bounds) + } + } + } + }) + } + const manager = new WalletStorageManager(identity, source, [destination]) + await manager.syncToWriter({ identityKey: identity }, destination) + await manager.updateBackups() + await manager.syncFromReader(identity, destination) + expect(limits.length).toBeGreaterThanOrEqual(36) + expect(limits.every(limit => limit === 10000000)).toBe(true) + const fresh = new WalletStorageManager(identity, source) + expect((await fresh.syncToWriterResumable({ identityKey: identity }, destination)).status).toBe('completed') +}) + +test('unmigrated destinations refuse durable sessions without applying an implicit migration', async () => { + const { source, destination } = await fixture(0) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const capability = destination.getSnapshotSync()! + await destination.knex.schema.dropTable('knex_migrations') + expect(await capability.supportsDestination()).toBe(false) + await expect(capability.begin(view, 'source')).rejects.toThrow('requires the version-one migration') + expect(await destination.knex.schema.hasTable('knex_migrations')).toBe(false) + await view.close() +}) + +test.each([-1, 9007199254740992])('invalid persisted primary epoch %s refuses a session', async epoch => { + const { source, destination } = await fixture(0) + const view = (await source.getSnapshotSync()!.openSource(identity))! + await destination.knex('snapshot_sync_primary_epochs').update({ epoch }) + await expect(destination.getSnapshotSync()!.begin(view, 'source')).rejects.toThrow('Invalid primary epoch') + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + await view.close() +}) + +test('source expiry during destination admission creates no durable session', async () => { + const { source, destination } = await fixture(0) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const original = destination.findOrInsertUser.bind(destination) + let clock: jest.SpyInstance | undefined + jest.spyOn(destination, 'findOrInsertUser').mockImplementation(async (...args) => { + const result = await original(...args) + clock = jest.spyOn(Date, 'now').mockReturnValue(view.expiresAt) + return result + }) + try { + await expect(destination.getSnapshotSync()!.begin(view, 'source')).rejects.toThrow( + 'expired before session admission' + ) + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + } finally { + clock?.mockRestore() + await view.close() + } +}) + +test.each(['changed', 'removed'] as const)( + 'a proof %s after preparation cannot advance a snapshot page', + async change => { + const { source, destination, user } = await fixture(0) + const foreign = (await source.findUserByIdentityKey(foreignIdentity))! + await seedClosure(source, user.userId, foreign.userId) + const proof = (await source.findProvenTxs({ partial: { provenTxId: 1 } }))[0] + await destination.insertProvenTx({ ...proof, provenTxId: 0 }) + const prior = (await destination.findProvenTxs({ partial: { txid: proof.txid } }))[0] + const second = (await source.findProvenTxs({ partial: { provenTxId: 3 } }))[0] + await destination.insertProvenTx({ ...second, provenTxId: 0 }) + const proofs = await destination.findProvenTxs({ partial: {} }) + const view = (await source.getSnapshotSync()!.openSource(identity))! + // Isolate the commit-time compare-and-set from the separately tested canonical + // proof service: this callback supplies its captured preflight authority. + const writer = new KnexSnapshotSyncDestination(destination, async () => new Map(proofs.map(row => [row.txid, row]))) + const before = await writer.begin(view, 'source') + const apply = await writer.prepare(before, await view.readPage('provenTxs')) + if (change === 'removed') await destination.knex('proven_txs').where({ provenTxId: prior.provenTxId }).del() + else await destination.updateProvenTx(prior.provenTxId, { blockHash: 'c'.repeat(64) }) + await expect(apply()).rejects.toThrow('Proof changed during snapshot preparation') + expect(await writer.checkpoint(identity, 'source')).toEqual(before) + expect(await destination.knex('snapshot_sync_ids')).toHaveLength(0) + await view.close() + } +) + +test('a source capability returning the wrong profile is closed before any destination mutation', async () => { + const { source, destination, manager } = await fixture(1) + const capability = source.getSnapshotSync()! + const writer = destination.getSnapshotSync()! + const begin = jest.fn(writer.begin) + let opened: WalletReadSnapshot | undefined + jest.spyOn(destination, 'getSnapshotSync').mockReturnValue({ ...writer, begin }) + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ + ...capability, + openSource: async (...args) => { + opened = (await capability.openSource(...args))! + return { ...opened, user: { ...opened.user, identityKey: foreignIdentity } } + } + }) + await expect(manager.syncToWriterResumable({ identityKey: identity }, destination)).rejects.toBeInstanceOf( + WERR_UNAUTHORIZED + ) + expect(begin).not.toHaveBeenCalled() + expect(opened!.isOpen).toBe(false) + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) +}) + +test('a cleanup failure remains observable after the destination commits the complete copy', async () => { + const { source, destination, manager } = await fixture(1) + const capability = source.getSnapshotSync()! + const failure = new Error('synthetic physical cleanup failure') + const legacy = jest.spyOn(destination, 'processSyncChunk') + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ + ...capability, + openSource: async (...args) => { + const view = (await capability.openSource(...args))! + return { + ...view, + close: async () => { + await view.close() + throw failure + } + } + } + }) + await expect(manager.syncToWriterResumable({ identityKey: identity }, destination)).rejects.toBe(failure) + expect((await destination.getSnapshotSync()!.checkpoint(identity, 'source'))!.done).toBe(true) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(1) + expect(legacy).not.toHaveBeenCalled() +}) + +test.each([ + ['sourceUserId', 999], + ['sourceActiveStorage', 'different-primary'], + ['sourceUserCreatedAt', '2000-01-01T00:00:00.000Z'], + ['sourceUserUpdatedAt', '2000-01-01T00:00:00.000Z'], + ['chain', 'main'], + ['destinationStorageIdentityKey', 'different-destination'], + ['version', 2], + ['activeStorage', 'different-primary'], + ['expiresAt', 1], + ['primaryEpoch', 99] +])('same-view admission refuses a changed durable %s binding', async (field, value) => { + const { source, destination } = await fixture(0) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + try { + await writer.begin(view, 'source') + await destination.knex('snapshot_sync_sessions').update({ [field]: value }) + const before = await destination.knex('snapshot_sync_sessions').first() + await expect(writer.begin(view, 'source')).rejects.toThrow('Snapshot session binding changed') + expect(await destination.knex('snapshot_sync_sessions').first()).toEqual(before) + } finally { + await view.close() + } +}) + +test.each([ + ['version', 2], + ['sessionId', 'g'.repeat(64)], + ['sessionId', 'x' + 'a'.repeat(64)], + ['sessionId', 'a'.repeat(64) + 'x'], + ['sequence', -1], + ['sequence', 0.5], + ['sequence', Number.MAX_SAFE_INTEGER + 1], + ['tableIndex', -1], + ['tableIndex', 0.5] +])('malformed checkpoint %s=%s rejects before proof preparation', async (field, value) => { + const { source, destination } = await fixture(0) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const proofs = jest.fn(async () => new Map()) + const writer = new KnexSnapshotSyncDestination(destination, proofs) + try { + const checkpoint = await writer.begin(view, 'source') + await expect( + writer.prepare({ ...checkpoint, [field]: value }, { rows: [], done: true, payloadBytes: 0 }) + ).rejects.toThrow('a version-one durable snapshot checkpoint') + expect(proofs).not.toHaveBeenCalled() + expect(await writer.checkpoint(identity, 'source')).toEqual(checkpoint) + } finally { + await view.close() + } +}) + +test.each(['snapshot-prefix', 'snapshot-suffix', 'identity-prefix', 'identity-suffix', 'storage-type'])( + 'source %s rejects before creating a profile or session', + async kind => { + const { source, destination } = await fixture(0) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + const input = { ...view, user: { ...view.user }, sourceStorage: { ...view.sourceStorage } } + if (kind === 'snapshot-prefix') input.snapshotId = 'x' + view.snapshotId + if (kind === 'snapshot-suffix') input.snapshotId = view.snapshotId + 'x' + if (kind === 'identity-prefix') input.user.identityKey = 'x' + identity + if (kind === 'identity-suffix') input.user.identityKey = identity + 'x' + if (kind === 'storage-type') input.sourceStorage.storageIdentityKey = 1 as unknown as string + try { + await expect(writer.begin(input, 'source')).rejects.toThrow('a live version-one wallet snapshot') + expect(await destination.findUsers({ partial: {} })).toHaveLength(1) + expect(await destination.knex('snapshot_sync_sessions')).toHaveLength(0) + } finally { + await view.close() + } + } +) + +test.each([ + ['version', 2], + ['identityKey', foreignIdentity], + ['snapshotId', 'c'.repeat(64)], + ['tableIndex', 2], + ['sequence', 99], + ['sessionId', 'c'.repeat(64)], + ['activeStorage', 'another-primary'], + ['chain', 'main'], + ['cursor-version', 2], + ['cursor-snapshotId', 'c'.repeat(64)], + ['cursor-table', 'outputs'], + ['cursor-after', [999]] +])('a prepared page refuses changed durable position %s before applying rows', async (field, value) => { + const { source, destination } = await fixture(3) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const writer = destination.getSnapshotSync()! + try { + let checkpoint = await advance(view, writer, 3) + const first = await view.readPage('txLabels', checkpoint.cursor, { maxRows: 1 }) + checkpoint = (await (await writer.prepare(checkpoint, first))()).checkpoint + const page = await view.readPage('txLabels', checkpoint.cursor, { maxRows: 1 }) + const apply = await writer.prepare(checkpoint, page) + const patch = field.startsWith('cursor-') + ? { cursor: JSON.stringify({ ...checkpoint.cursor, [field.slice(7)]: value }) } + : { [field]: value } + await destination.knex('snapshot_sync_sessions').update(patch) + const prior = await destination.knex('snapshot_sync_sessions').first() + await expect(apply()).rejects.toThrow('Snapshot session changed') + expect(await destination.knex('snapshot_sync_sessions').first()).toEqual(prior) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(1) + } finally { + await view.close() + } +}) + +test('source admission accepts the exact lifetime and storage-identity length ceilings', async () => { + const { source, destination } = await fixture(0) + const view = (await source.getSnapshotSync()!.openSource(identity))! + const now = Date.now() + const clock = jest.spyOn(Date, 'now').mockReturnValue(now) + const storageIdentityKey = 'a'.repeat(130) + try { + const result = await destination.getSnapshotSync()!.begin( + { + ...view, + expiresAt: now + 3600000, + sourceStorage: { ...view.sourceStorage, storageIdentityKey } + }, + 'source' + ) + expect(result.sourceStorageIdentityKey).toBe(storageIdentityKey) + expect(result.sequence).toBe(0) + } finally { + clock.mockRestore() + await view.close() + } +}) + +test.each(['push', 'backup', 'legacy-backup'] as const)( + 'ordinary %s delivers committed-page progress to the existing logger', + async operation => { + const { source, destination, manager } = await fixture(2, operation !== 'legacy-backup') + const messages: string[] = [] + const rendered: string[] = [] + const logger = (message: string): string => { + messages.push(message) + const value = `[observed:${message}]` + rendered.push(value) + return value + } + const log = + operation === 'push' + ? (await manager.syncToWriter(await manager.getAuth(), destination, undefined, 'prefix:', logger)).log + : await manager.updateBackups(undefined, logger) + const pages = messages.filter(message => message.startsWith('chunk ')) + expect(pages.length).toBeGreaterThan(0) + expect(messages.indexOf(pages[0])).toBeLessThan(messages.length - 1) + for (const value of rendered) expect(log).toContain(value) + if (operation === 'push') expect(log.startsWith('prefix:')).toBe(true) + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(2) + expect(source.getSnapshotSync()).toBeDefined() + } +) + +test('snapshot capability configuration rejects a non-boolean before database admission', async () => { + const database = knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) + try { + expect( + () => + new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: database, + snapshotSync: 'enabled' as unknown as boolean + }) + ).toThrow('snapshotSync') + expect(database.client.pool.numUsed()).toBe(0) + } finally { + await database.destroy() + } +}) + +test('an expiry during cleanup preserves an earlier malformed-page failure', async () => { + const { source, destination, manager } = await fixture(1) + const capability = source.getSnapshotSync()! + const failure = new Error('synthetic malformed page') + const legacy = jest.spyOn(destination, 'processSyncChunk') + jest.spyOn(source, 'getSnapshotSync').mockReturnValue({ + ...capability, + openSource: async (...args) => { + const view = (await capability.openSource(...args))! + return { + ...view, + readPage: async () => { + throw failure + }, + close: async () => { + await view.close() + throw new SnapshotResourceLimitError('synthetic cleanup expiry') + } + } + } + }) + await expect(manager.syncToWriterResumable({ identityKey: identity }, destination)).rejects.toBe(failure) + expect(legacy).not.toHaveBeenCalled() + expect(await destination.findTxLabels({ partial: {} })).toHaveLength(0) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.property.test.ts new file mode 100644 index 000000000..2bc340075 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.property.test.ts @@ -0,0 +1,244 @@ +import fc from 'fast-check' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import { snapshotSyncTables } from './SnapshotSync' +import { WalletStorageManager } from '../WalletStorageManager' +import type { WalletStorageProvider } from '../../sdk/WalletStorage.interfaces' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +test('random durable page schedules recover acknowledgements and restarts without profile leakage or duplicate rows', async () => { + const directory = await mkdtemp(join(tmpdir(), 'wallet-durable-property-')) + const open = (name: string) => + new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, name + '.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) + const source = open('source') + const destination = open('destination') + const identity = '02' + '11'.repeat(32) + const foreignIdentity = '03' + '22'.repeat(32) + try { + for (const [store, name] of [ + [source, 'source'], + [destination, 'destination'] + ] as const) { + await store.knex.raw('PRAGMA journal_mode = WAL') + await store.migrate(name, name) + await store.makeAvailable() + } + const { user } = await source.findOrInsertUser(identity) + const { user: foreign } = await source.findOrInsertUser(foreignIdentity) + const { user: occupied } = await destination.findOrInsertUser(foreignIdentity) + const { user: target } = await destination.findOrInsertUser(identity) + await source.updateUser(user.userId, { activeStorage: 'source' }) + await destination.updateUser(target.userId, { activeStorage: 'source' }) + const writer = destination.getSnapshotSync()! + await fc.assert( + fc.asyncProperty( + fc.array(fc.record({ foreign: fc.boolean(), deleted: fc.boolean(), length: fc.integer({ min: 0, max: 80 }) }), { + maxLength: 12 + }), + fc.integer({ min: 1, max: 6 }), + fc.integer({ min: 2048, max: 8192 }), + fc.array(fc.constantFrom('commit', 'lost-ack', 'stale', 'restart', 'discard'), { maxLength: 8 }), + async (entries, maxRows, maxBytes, steps) => { + await destination.knex('snapshot_sync_ids').del() + await destination.knex('snapshot_sync_sessions').del() + await destination.knex('tx_labels').del() + await source.knex('tx_labels').del() + const date = new Date('2026-01-01T00:00:00.000Z') + const records = entries.map((entry, index) => ({ + txLabelId: index + 1, + userId: entry.foreign ? foreign.userId : user.userId, + label: `label-${index}-${'é'.repeat(entry.length)}`, + isDeleted: entry.deleted, + created_at: date.toISOString(), + updated_at: date.toISOString() + })) + if (records.length > 0) await source.knex('tx_labels').insert(records) + await destination.findOrInsertTxLabel(occupied.userId, 'occupied') + let view = (await source.getSnapshotSync()!.openSource(identity))! + let checkpoint = await writer.begin(view, view.user.activeStorage) + let restarted = false + let step = 0 + const after = { + created_at: date.toISOString(), + updated_at: new Date(date.getTime() + 1).toISOString(), + userId: user.userId, + label: 'after-view', + isDeleted: true + } + await source.knex('tx_labels').insert(after) + try { + while (!checkpoint.done) { + expect(step).toBeLessThan(80) + const action = steps[step++] ?? 'commit' + const table = snapshotSyncTables[checkpoint.tableIndex] + const page = await view.readPage(table, checkpoint.cursor, { maxRows, maxBytes }) + expect(page.rows.length).toBeLessThanOrEqual(maxRows) + expect(page.payloadBytes).toBeLessThanOrEqual(maxBytes) + const apply = await writer.prepare(checkpoint, page) + if (action === 'restart' && !restarted) { + await view.close() + view = (await source.getSnapshotSync()!.openSource(identity))! + checkpoint = await writer.begin(view, view.user.activeStorage) + expect(checkpoint.sequence).toBe(0) + await expect(apply()).rejects.toThrow('session changed') + restarted = true + continue + } + if (action === 'discard') { + expect(await writer.checkpoint(identity, 'source')).toEqual(checkpoint) + continue + } + const stale = action === 'stale' ? await writer.prepare(checkpoint, page) : undefined + const acknowledged = await apply() + const durable = (await writer.checkpoint(identity, 'source'))! + expect(durable.sequence).toBe(checkpoint.sequence + 1) + expect(durable).toEqual(acknowledged.checkpoint) + if (stale !== undefined) await expect(stale()).rejects.toThrow('session changed') + // A discarded acknowledgement and an observed one converge on the + // same destination record, independent of a sender progress log. + checkpoint = action === 'lost-ack' ? durable : acknowledged.checkpoint + } + const project = (row: { label: string; isDeleted: boolean }) => ({ + label: row.label, + isDeleted: row.isDeleted + }) + const order = (a: { label: string }, b: { label: string }) => a.label.localeCompare(b.label) + const expected = records.filter(row => row.userId === user.userId).map(project) + if (restarted) expected.push(project(after)) + const actual = await destination.findTxLabels({ partial: { userId: target.userId } }) + expect(actual.map(project).sort(order)).toEqual(expected.sort(order)) + expect(new Set(actual.map(row => row.txLabelId)).size).toBe(expected.length) + expect(await destination.findTxLabels({ partial: { userId: occupied.userId } })).toHaveLength(1) + const mappings = await destination.knex('snapshot_sync_ids').where({ entity: 'txLabel' }) + expect(mappings).toHaveLength(expected.length) + expect( + mappings.every(row => row.userId === target.userId && row.sourceStorageIdentityKey === 'source') + ).toBe(true) + expect(await destination.knex('sync_states')).toHaveLength(0) + } finally { + await view.close() + } + } + ) + ) + } finally { + await source.destroy() + await destination.destroy() + await rm(directory, { recursive: true, force: true }) + } +}, 120000) + +function deferred() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} + +test('primary requests retain queue order through a delayed success or failure', async () => { + await fc.assert( + fc.asyncProperty( + fc.array(fc.constantFrom('A', 'B', 'C'), { minLength: 1, maxLength: 12 }), + fc.boolean(), + async (requests, failFirst) => { + const entered = deferred(), + release = deferred(), + failure = new Error('synthetic first primary switch failure') + let first = true + function provider(key: string, userId: number) { + const settings = { + chain: 'test', + storageIdentityKey: key, + storageName: key, + dbtype: 'SQLite', + maxOutputScriptLength: 1024 + } + const user = { userId, identityKey: 'synthetic-primary-order', activeStorage: 'A' } + return { + user, + isStorageProvider: () => false, + makeAvailable: async () => settings, + getSettings: () => settings, + findOrInsertUser: async () => ({ user: { ...user } }), + setActive: async (_auth: unknown, activeStorage: string) => { + if (first) { + first = false + entered.resolve() + await release.promise + if (failFirst) throw failure + } + user.activeStorage = activeStorage + } + } + } + const providers = ['A', 'B', 'C'].map((key, index) => provider(key, index + 1)) + const manager = new WalletStorageManager( + 'synthetic-primary-order', + providers[0] as unknown as WalletStorageProvider, + providers.slice(1) as unknown as WalletStorageProvider[] + ) + await manager.makeAvailable() + manager.syncToWriter = async (_auth, writer, source) => { + const target = providers.find( + p => p.getSettings().storageIdentityKey === writer.getSettings().storageIdentityKey + )! + const origin = providers.find( + p => p.getSettings().storageIdentityKey === source!.getSettings().storageIdentityKey + )! + target.user.activeStorage = origin.user.activeStorage + return { inserts: 0, updates: 0, log: '' } + } + const switching = manager.setActive('B') + const checkedSwitch = failFirst ? expect(switching).rejects.toBe(failure) : switching + await entered.promise + let completed = 0 + const queued = requests.map(key => + manager.setActive(key).then(() => { + completed++ + }) + ) + try { + await new Promise(resolve => setImmediate(resolve)) + expect(completed).toBe(0) + } finally { + release.resolve() + await checkedSwitch + await Promise.all(queued) + } + const expected = requests[requests.length - 1] + expect(manager.getActiveStore()).toBe(expected) + expect(manager.isActiveEnabled).toBe(true) + expect(providers.map(p => p.user.activeStorage)).toEqual([expected, expected, expected]) + } + ), + { + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + seed: Number.isSafeInteger(requestedSeed) ? requestedSeed : 3242026, + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}), + interruptAfterTimeLimit: 150000, + markInterruptAsFailure: true + } + ) +}, 180000) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.ts new file mode 100644 index 000000000..b19179194 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSync.ts @@ -0,0 +1,72 @@ +import type { TableSettings, TableUser } from '../schema/tables' +import type { + WalletReadSnapshot, + WalletReadSnapshotOptions, + WalletSnapshotCursor, + WalletSnapshotPage +} from './WalletReadSnapshot' + +/** Dependency order for replica merges. Source operational sync states are archive provenance, not replica cursors. */ +export const snapshotSyncTables = Object.freeze([ + 'provenTxs', + 'outputBaskets', + 'outputTags', + 'txLabels', + 'transactions', + 'outputs', + 'txLabelMaps', + 'outputTagMaps', + 'certificates', + 'certificateFields', + 'commissions', + 'provenTxReqs' +] as const) +export type SnapshotSyncTable = (typeof snapshotSyncTables)[number] + +/** Version one binds the packed row schema as well as the source view and profile. Local only. */ +export interface SnapshotSyncSource { + version: 1 + snapshotId: string + sourceStorage: TableSettings + user: TableUser + expiresAt: number +} + +export interface SnapshotSyncCheckpoint { + version: 1 + sessionId: string + identityKey: string + sourceStorageIdentityKey: string + destinationStorageIdentityKey: string + snapshotId: string + /** Next table in snapshotSyncTables. Its length means complete. */ + tableIndex: number + sequence: number + cursor?: WalletSnapshotCursor + done: boolean +} + +export interface SnapshotSyncCommit { + checkpoint: SnapshotSyncCheckpoint + inserts: number + updates: number +} + +/** Additive provider capability. These adapter methods are absent from the RPC allowlist. */ +export interface SnapshotSyncStorage { + /** False prevents switching away from an accepted immutable remote view on a resource error. */ + fallbackOnResourceError?: boolean + /** True only after the version-one auxiliary schema and primary fencing migration committed. */ + supportsDestination: () => Promise + /** Undefined means this configuration cannot retain a view while foreground writes proceed. */ + openSource: (identityKey: string, options?: WalletReadSnapshotOptions) => Promise + /** Same-view retry resumes the destination cursor. A different view atomically fences its predecessor and starts at table zero. */ + begin: (source: SnapshotSyncSource, activeStorage: string | undefined) => Promise + /** Observe the durable outcome after a lost acknowledgement. Never infer progress from a sender log. */ + checkpoint: (identityKey: string, sourceStorageIdentityKey: string) => Promise + /** Detaches input and completes proof I/O before returning a one-use atomic merge/checkpoint operation. */ + prepare: ( + checkpoint: SnapshotSyncCheckpoint, + page: WalletSnapshotPage + ) => Promise<() => Promise> +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncDestination.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncDestination.property.test.ts new file mode 100644 index 000000000..486a10667 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncDestination.property.test.ts @@ -0,0 +1,73 @@ +import fc from 'fast-check' +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageProvider } from '../StorageProvider' +import type { SnapshotSyncSource } from './SnapshotSync' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +test('random profile bindings reject same-view drift without changing durable checkpoints or another profile', async () => { + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) + }) + const identities = ['02' + '11'.repeat(32), '03' + '22'.repeat(32)] + try { + await storage.migrate('destination', 'destination') + await storage.makeAvailable() + const writer = storage.getSnapshotSync()! + const users = [] + for (const identity of identities) users.push((await storage.findOrInsertUser(identity)).user) + await fc.assert( + fc.asyncProperty( + fc.boolean(), + fc.integer({ min: 1, max: 1000000 }), + fc.integer({ min: 1, max: 1000000 }), + fc.constantFrom('userId', 'created_at', 'updated_at', 'activeStorage', 'expiresAt'), + async (second, sourceId, epoch, field) => { + await storage.knex('snapshot_sync_sessions').delete() + const selected = Number(second) + const date = new Date(1600000000000 + epoch) + const source: SnapshotSyncSource = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceStorage: { ...storage.getSettings(), storageIdentityKey: 'source' }, + user: { ...users[selected], userId: sourceId, created_at: date, updated_at: date, activeStorage: 'source' }, + expiresAt: Date.now() + 600000 + } + const other: SnapshotSyncSource = { + ...source, + user: { ...source.user, identityKey: identities[1 - selected] } + } + const otherCheckpoint = await writer.begin(other, 'source') + const checkpoint = await writer.begin(source, 'source') + expect(checkpoint.identityKey).toBe(identities[selected]) + expect(await writer.begin(source, 'source')).toEqual(checkpoint) + const changed = { ...source, user: { ...source.user } } + if (field === 'userId') changed.user.userId++ + else if (field === 'created_at' || field === 'updated_at') changed.user[field] = new Date(date.getTime() + 1) + else if (field === 'activeStorage') changed.user.activeStorage = 'changed-primary' + else changed.expiresAt++ + await expect(writer.begin(changed, changed.user.activeStorage)).rejects.toThrow('binding changed') + expect(await writer.checkpoint(identities[selected], 'source')).toEqual(checkpoint) + expect(await writer.checkpoint(identities[1 - selected], 'source')).toEqual(otherCheckpoint) + expect(await storage.knex('snapshot_sync_ids')).toHaveLength(0) + const replacement = await writer.begin({ ...source, snapshotId: 'b'.repeat(64) }, 'source') + expect(replacement).toMatchObject({ snapshotId: 'b'.repeat(64), sequence: 0, tableIndex: 0, done: false }) + expect(replacement.sessionId).not.toBe(checkpoint.sessionId) + expect(await writer.checkpoint(identities[1 - selected], 'source')).toEqual(otherCheckpoint) + } + ) + ) + } finally { + await storage.destroy() + } +}, 120000) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.property.test.ts new file mode 100644 index 000000000..03b799b75 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.property.test.ts @@ -0,0 +1,93 @@ +import fc from 'fast-check' +import { knex } from 'knex' +import { loadSnapshotIdMap } from './SnapshotSyncRows' +import type { SyncChunk } from '../../sdk/WalletStorage.interfaces' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +test('random overlapping parent IDs preserve exact profile/source maps and persist only learned child IDs', async () => { + const database = knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) + try { + await database.schema.createTable('snapshot_sync_ids', table => { + table.integer('userId') + table.string('sourceStorageIdentityKey') + table.string('entity') + table.integer('incomingId') + table.integer('localId') + table.primary(['userId', 'sourceStorageIdentityKey', 'entity', 'incomingId']) + }) + await fc.assert( + fc.asyncProperty( + fc.uniqueArray(fc.integer({ min: 1, max: 100000 }), { minLength: 1, maxLength: 12 }), + fc.integer({ min: 1, max: 10000 }), + fc.boolean(), + async (incoming, userId, second) => { + await database('snapshot_sync_ids').delete() + const scope = { userId, sourceStorageIdentityKey: second ? 'second' : 'first' } + const original = incoming.flatMap((id, index) => [ + { ...scope, entity: 'transaction', incomingId: id, localId: index + 101 }, + { ...scope, userId: userId + 1, entity: 'transaction', incomingId: id, localId: index + 201 }, + { + ...scope, + sourceStorageIdentityKey: second ? 'first' : 'second', + entity: 'transaction', + incomingId: id, + localId: index + 301 + } + ]) + await database('snapshot_sync_ids').insert(original) + const chunk = { + userIdentityKey: 'identity', + fromStorageIdentityKey: scope.sourceStorageIdentityKey, + toStorageIdentityKey: 'destination', + outputs: incoming.map(id => ({ userId: 77, outputId: id, transactionId: id, spentBy: id })) + } as SyncChunk + const loaded = await loadSnapshotIdMap(database, scope, 77, 'outputs', chunk) + expect(loaded.map.transaction.idMap).toEqual( + Object.fromEntries(incoming.map((id, index) => [id, index + 101])) + ) + expect(loaded.map.output.idMap).toEqual({}) + for (const [index, id] of incoming.entries()) loaded.map.output.idMap[id] = index + 401 + await loaded.persist() + expect( + await database('snapshot_sync_ids') + .where({ entity: 'transaction' }) + .orderBy(['userId', 'sourceStorageIdentityKey', 'incomingId']) + ).toEqual( + [...original].sort( + (a, b) => + a.userId - b.userId || + a.sourceStorageIdentityKey.localeCompare(b.sourceStorageIdentityKey) || + a.incomingId - b.incomingId + ) + ) + expect( + await database('snapshot_sync_ids') + .where({ ...scope, entity: 'output' }) + .orderBy('incomingId') + ).toEqual( + incoming + .map((id, index) => ({ ...scope, entity: 'output', incomingId: id, localId: index + 401 })) + .sort((a, b) => a.incomingId - b.incomingId) + ) + expect(await database('snapshot_sync_ids')).toHaveLength(incoming.length * 4) + const missing = { ...chunk, outputs: [{ userId: 77, outputId: 100001, transactionId: 100001 }] } as SyncChunk + await expect(loadSnapshotIdMap(database, scope, 77, 'outputs', missing)).rejects.toThrow( + 'parent mapping is missing' + ) + expect(await database('snapshot_sync_ids')).toHaveLength(incoming.length * 4) + } + ) + ) + } finally { + await database.destroy() + } +}, 120000) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.test.ts new file mode 100644 index 000000000..cd3a3f9c4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.test.ts @@ -0,0 +1,279 @@ +import { knex, type Knex } from 'knex' +import { loadSnapshotIdMap, detachSnapshotSyncPage } from './SnapshotSyncRows' +import type { SyncChunk } from '../../sdk/WalletStorage.interfaces' +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import type { SnapshotSyncCheckpoint, SnapshotSyncTable } from './SnapshotSync' +import type { WalletSnapshotPage } from './WalletReadSnapshot' + +let database: Knex +const scope = { userId: 10, sourceStorageIdentityKey: 'source' } +beforeEach(async () => { + database = knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) + await database.schema.createTable('snapshot_sync_ids', table => { + table.integer('userId') + table.string('sourceStorageIdentityKey') + table.string('entity') + table.integer('incomingId') + table.integer('localId') + table.primary(['userId', 'sourceStorageIdentityKey', 'entity', 'incomingId']) + }) +}) +afterEach(async () => { + await database.destroy() +}) +function chunk(table: SnapshotSyncTable, rows: unknown[]): SyncChunk { + return { + userIdentityKey: 'identity', + fromStorageIdentityKey: 'source', + toStorageIdentityKey: 'destination', + [table]: rows + } as SyncChunk +} + +test('ID lookup is profile/source scoped, maps parents and persists only newly learned IDs', async () => { + await database('snapshot_sync_ids').insert([ + { ...scope, entity: 'transaction', incomingId: 1, localId: 101 }, + { ...scope, entity: 'outputBasket', incomingId: 2, localId: 102 }, + { ...scope, entity: 'output', incomingId: 3, localId: 103 }, + { ...scope, userId: 20, entity: 'transaction', incomingId: 1, localId: 999 }, + { ...scope, sourceStorageIdentityKey: 'other', entity: 'transaction', incomingId: 1, localId: 888 } + ]) + const loaded = await loadSnapshotIdMap( + database, + scope, + 7, + 'outputs', + chunk('outputs', [ + { outputId: 3, userId: 7, transactionId: 1, basketId: 2, spentBy: 1 }, + { outputId: 4, userId: 7, transactionId: 1, basketId: 2 } + ]) + ) + expect(loaded.map.transaction.idMap).toEqual({ 1: 101 }) + expect(loaded.map.outputBasket.idMap).toEqual({ 2: 102 }) + expect(loaded.map.output.idMap).toEqual({ 3: 103 }) + loaded.map.output.idMap[4] = 104 + await loaded.persist() + expect( + await database('snapshot_sync_ids') + .where({ ...scope, entity: 'output' }) + .orderBy('incomingId') + ).toEqual([ + { ...scope, entity: 'output', incomingId: 3, localId: 103 }, + { ...scope, entity: 'output', incomingId: 4, localId: 104 } + ]) + expect(await database('snapshot_sync_ids')).toHaveLength(6) +}) + +test('mapping references are capped before SQL, while the exact bound permits bounded lookup batches', async () => { + const notify = (count: number) => + chunk('provenTxReqs', [ + { provenTxReqId: 1, notify: JSON.stringify({ transactionIds: Array.from({ length: count }, (_, n) => n + 1) }) } + ]) + const queries: Array<{ sql: string; bindings: unknown[] }> = [] + database.on('query', query => queries.push(query)) + await expect(loadSnapshotIdMap(database, scope, 7, 'provenTxReqs', notify(4096))).rejects.toBeInstanceOf( + SnapshotResourceLimitError + ) + expect(queries).toHaveLength(0) + const accepted = await loadSnapshotIdMap(database, scope, 7, 'provenTxReqs', notify(4095)) + expect(queries).toHaveLength(33) + expect(queries.every(query => query.bindings.length <= 131)).toBe(true) + // Unmapped notifications can refer to another profile and are not fabricated. + expect(accepted.map.transaction.idMap).toEqual({}) + await accepted.persist() + expect(queries).toHaveLength(33) +}) + +test('missing parent mappings and invalid notification IDs cannot produce a partial map', async () => { + await expect( + loadSnapshotIdMap( + database, + scope, + 7, + 'transactions', + chunk('transactions', [{ userId: 7, transactionId: 1, provenTxId: 2 }]) + ) + ).rejects.toThrow('parent mapping is missing') + for (const transactionIds of [1, {}, [0], [-1], [1.5], ['1']]) + await expect( + loadSnapshotIdMap( + database, + scope, + 7, + 'provenTxReqs', + chunk('provenTxReqs', [{ provenTxReqId: 1, notify: JSON.stringify({ transactionIds }) }]) + ) + ).rejects.toThrow() + expect(await database('snapshot_sync_ids')).toHaveLength(0) +}) + +test('packed row admission refuses non-record rows and extra allocation before copying', () => { + const checkpoint: SnapshotSyncCheckpoint = { + version: 1, + sessionId: 'a'.repeat(64), + snapshotId: 'b'.repeat(64), + identityKey: 'identity', + sourceStorageIdentityKey: 'source', + destinationStorageIdentityKey: 'destination', + tableIndex: 3, + sequence: 3, + done: false + } + const page = (row: unknown): WalletSnapshotPage<'txLabels'> => + ({ + rows: [row], + payloadBytes: 8192, + done: true, + cursor: { version: 1, snapshotId: checkpoint.snapshotId, table: 'txLabels', after: [1] } + }) as WalletSnapshotPage<'txLabels'> + for (const row of [null, 1, Object.fromEntries(Array.from({ length: 65 }, (_, n) => ['column' + n, n]))]) + expect(() => detachSnapshotSyncPage(checkpoint, page(row))).toThrow('flat snapshot records') + expect(() => + detachSnapshotSyncPage(checkpoint, { + rows: [], + payloadBytes: 0, + done: true, + cursor: { version: 1, snapshotId: checkpoint.snapshotId, table: 'txLabels', after: [1] } + }) + ).toThrow('unchanged empty-page') + const prior = { version: 1 as const, snapshotId: checkpoint.snapshotId, table: 'txLabels' as const, after: [1] } + expect( + detachSnapshotSyncPage({ ...checkpoint, cursor: prior }, { rows: [], payloadBytes: 0, done: true, cursor: prior }) + ).toMatchObject({ nextTable: 4, nextCursor: null }) + expect(() => detachSnapshotSyncPage({ ...checkpoint, cursor: prior }, page({ txLabelId: 1 }))).toThrow('last row') +}) + +const boundedCheckpoint: SnapshotSyncCheckpoint = { + version: 1, + sessionId: 'a'.repeat(64), + snapshotId: 'b'.repeat(64), + identityKey: 'identity', + sourceStorageIdentityKey: 'source', + destinationStorageIdentityKey: 'destination', + tableIndex: 3, + sequence: 3, + done: false +} +function boundedPage(rows: unknown[], payloadBytes = 16777216): WalletSnapshotPage<'txLabels'> { + const last = rows.at(-1) as { txLabelId: number } | undefined + return { + rows, + done: true, + payloadBytes, + ...(last === undefined + ? {} + : { + cursor: { version: 1, snapshotId: boundedCheckpoint.snapshotId, table: 'txLabels', after: [last.txLabelId] } + }) + } as WalletSnapshotPage<'txLabels'> +} + +test('archive positions are bounded before persistence while legacy cursor bytes stay unchanged', () => { + const page = { ...boundedPage([{ txLabelId: 1 }]), done: false } + expect(detachSnapshotSyncPage(boundedCheckpoint, page).nextCursor).toBe(JSON.stringify(page.cursor)) + const position = { version: 1 as const, archiveId: 'c'.repeat(64), sequence: 3, rowOffset: 1 } + page.cursor!.archivePosition = position + const persisted = detachSnapshotSyncPage(boundedCheckpoint, page).nextCursor! + position.rowOffset = 2 + expect(JSON.parse(persisted).archivePosition.rowOffset).toBe(1) + for (const invalid of [ + { ...position, rowOffset: 0 }, + { ...position, sequence: 4096 }, + { ...position, extra: 1 } + ]) { + page.cursor!.archivePosition = invalid + expect(() => detachSnapshotSyncPage(boundedCheckpoint, page)).toThrow('cursor.archivePosition') + } +}) + +test.each([ + [NaN, 'finite numbers'], + [Infinity, 'finite numbers'], + [new Date(NaN), 'finite dates'], + [{ nested: true }, 'packed bytes, dates and scalar values'], + [[1, 2], 'packed bytes, dates and scalar values'] +])('allocation refuses unsupported value %p before cloning', (value, reason) => { + expect(() => detachSnapshotSyncPage(boundedCheckpoint, boundedPage([{ txLabelId: 1, value }]))).toThrow( + `The rows parameter must be ${reason}` + ) +}) + +test.each(['abc', new Uint8Array([1, 2, 3])])( + 'allocation charge covers each string or packed-byte payload %p', + value => { + // Two cells cost 128 bytes plus twice the three-unit payload. + const page = boundedPage([{ txLabelId: 1, value }], 134) + expect(detachSnapshotSyncPage(boundedCheckpoint, page).chunk.txLabels).toHaveLength(1) + expect(() => detachSnapshotSyncPage(boundedCheckpoint, { ...page, payloadBytes: 133 })).toThrow( + 'The payloadBytes parameter must be an allocation charge covering every row' + ) + } +) + +test('inclusive row, column and byte ceilings accept their exact bounds', () => { + const row = Object.fromEntries(Array.from({ length: 63 }, (_, n) => ['column' + n, n])) + expect( + detachSnapshotSyncPage(boundedCheckpoint, boundedPage([{ ...row, txLabelId: 1 }], 4096)).chunk.txLabels + ).toHaveLength(1) + const page = boundedPage(Array.from({ length: 1000 }, (_, n) => ({ txLabelId: n + 1 }))) + expect(detachSnapshotSyncPage(boundedCheckpoint, page).chunk.txLabels).toHaveLength(1000) +}) + +test.each([ + { rows: undefined }, + { rows: Array.from({ length: 1001 }, (_, n) => ({ txLabelId: n + 1 })) }, + { done: undefined }, + { payloadBytes: -1 }, + { payloadBytes: 0.5 }, + { payloadBytes: 16777217 }, + { rows: [], done: false } +])('invalid page envelope %p rejects at page admission', patch => { + expect(() => + detachSnapshotSyncPage(boundedCheckpoint, { ...boundedPage([]), ...patch } as WalletSnapshotPage<'txLabels'>) + ).toThrow('The page parameter must be a bounded snapshot page making progress') +}) + +test.each([{ tableIndex: 12 }, { done: true }])('complete checkpoints refuse pages before allocation: %p', patch => { + expect(() => detachSnapshotSyncPage({ ...boundedCheckpoint, ...patch }, boundedPage([]))).toThrow( + 'Snapshot sync is already complete' + ) +}) + +test('composite cursors must match every final-row key and the original source view', () => { + const checkpoint = { ...boundedCheckpoint, tableIndex: 6 } + const page: WalletSnapshotPage<'txLabelMaps'> = { + rows: [{ txLabelId: 1, transactionId: 2 } as never], + done: false, + payloadBytes: 128, + cursor: { version: 1, snapshotId: checkpoint.snapshotId, table: 'txLabelMaps', after: [1, 2] } + } + expect(detachSnapshotSyncPage(checkpoint, page).nextCursor).toBe(JSON.stringify(page.cursor)) + for (const after of [ + [1, 3], + [3, 2] + ]) + expect(() => detachSnapshotSyncPage(checkpoint, { ...page, cursor: { ...page.cursor!, after } })).toThrow( + 'The page.cursor parameter must be the last row of this source view and table' + ) + for (const cursor of [undefined, { ...page.cursor!, snapshotId: 'c'.repeat(64) }]) + expect(() => detachSnapshotSyncPage(checkpoint, { ...page, cursor })).toThrow( + 'The page.cursor parameter must be the last row of this source view and table' + ) +}) + +test('mapping batches never exceed 128 new IDs or include another profile in a global row', async () => { + const queries: Array<{ sql: string; bindings: unknown[] }> = [] + database.on('query', query => queries.push(query)) + const rows = Array.from({ length: 256 }, (_, n) => ({ userId: 7, txLabelId: n + 1 })) + const loaded = await loadSnapshotIdMap(database, scope, 7, 'txLabels', chunk('txLabels', rows)) + expect(queries).toHaveLength(2) + for (const row of rows) loaded.map.txLabel.idMap[row.txLabelId] = row.txLabelId + 1000 + await loaded.persist() + const inserts = queries.filter(query => query.sql.startsWith('insert')) + expect(inserts).toHaveLength(2) + expect(inserts.every(query => query.bindings.length === 128 * 5)).toBe(true) + expect(await database('snapshot_sync_ids')).toHaveLength(256) + await expect( + loadSnapshotIdMap(database, scope, 7, 'provenTxs', chunk('provenTxs', [{ provenTxId: 1, userId: 99 }])) + ).rejects.toThrow('Snapshot row belongs to another profile') +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.ts new file mode 100644 index 000000000..d3d3da400 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncRows.ts @@ -0,0 +1,252 @@ +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import type { Knex } from 'knex' +import type { SyncChunk } from '../../sdk/WalletStorage.interfaces' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +import { createSyncMap, type SyncMap } from '../schema/entities/EntityBase' +import { snapshotSyncPage } from '../sync/snapshotSyncPage' +import { snapshotSyncTables, type SnapshotSyncCheckpoint, type SnapshotSyncTable } from './SnapshotSync' +import type { WalletSnapshotPage } from './WalletReadSnapshot' +import { runInSeries } from '../../utility/runInSeries' +import { copySnapshotArchivePosition } from './SnapshotCursor' + +const entities: Record = { + provenTxs: 'provenTx', + outputBaskets: 'outputBasket', + outputTags: 'outputTag', + txLabels: 'txLabel', + transactions: 'transaction', + outputs: 'output', + txLabelMaps: 'txLabelMap', + outputTagMaps: 'outputTagMap', + certificates: 'certificate', + certificateFields: 'certificateField', + commissions: 'commission', + provenTxReqs: 'provenTxReq' +} +const ids: Record = { + provenTxId: 'provenTx', + basketId: 'outputBasket', + outputTagId: 'outputTag', + txLabelId: 'txLabel', + transactionId: 'transaction', + spentBy: 'transaction', + outputId: 'output', + certificateId: 'certificate', + commissionId: 'commission', + provenTxReqId: 'provenTxReq' +} +const keys: Record = { + provenTxs: ['provenTxId'], + outputBaskets: ['basketId'], + outputTags: ['outputTagId'], + txLabels: ['txLabelId'], + transactions: ['transactionId'], + outputs: ['outputId'], + txLabelMaps: ['txLabelId', 'transactionId'], + outputTagMaps: ['outputTagId', 'outputId'], + certificates: ['certificateId'], + certificateFields: ['fieldName', 'certificateId'], + commissions: ['commissionId'], + provenTxReqs: ['provenTxReqId'] +} + +function allocationCharge(value: unknown): number { + if (typeof value === 'string') return 64 + value.length * 2 + if (value instanceof Uint8Array) return 64 + value.byteLength * 2 + if (typeof value === 'number') { + if (!Number.isFinite(value)) throw new WERR_INVALID_PARAMETER('rows', 'finite numbers') + } else if (value != null && typeof value !== 'boolean') { + let time: number + try { + time = Date.prototype.getTime.call(value) + } catch { + throw new WERR_INVALID_PARAMETER('rows', 'packed bytes, dates and scalar values') + } + if (!Number.isFinite(time)) throw new WERR_INVALID_PARAMETER('rows', 'finite dates') + } + return 64 +} + +/** Recheck packed allocation before copying even when called without the built-in SQL reader. */ +function validateAllocation(rows: unknown[], advertisedBytes: number): void { + let charge = 0 + for (const row of rows) { + if (row === null || typeof row !== 'object' || Object.keys(row).length > 64) { + throw new WERR_INVALID_PARAMETER('rows', 'flat snapshot records with at most 64 columns') + } + for (const value of Object.values(row)) { + charge += allocationCharge(value) + if (charge > advertisedBytes) + throw new WERR_INVALID_PARAMETER('payloadBytes', 'an allocation charge covering every row') + } + } +} + +export function detachSnapshotSyncPage( + checkpoint: SnapshotSyncCheckpoint, + page: WalletSnapshotPage +): { + chunk: SyncChunk + nextCursor: string | null + nextTable: number +} { + const table = snapshotSyncTables[checkpoint.tableIndex] + if (table === undefined || checkpoint.done) throw new WERR_INVALID_OPERATION('Snapshot sync is already complete') + if ( + !Array.isArray(page.rows) || + page.rows.length > 1000 || + typeof page.done !== 'boolean' || + !Number.isSafeInteger(page.payloadBytes) || + page.payloadBytes < 0 || + page.payloadBytes > 16777216 || + (!page.done && page.rows.length === 0) + ) { + throw new WERR_INVALID_PARAMETER('page', 'a bounded snapshot page making progress') + } + validateAllocation(page.rows, page.payloadBytes) + const cursor = page.cursor + if (cursor?.archivePosition !== undefined) copySnapshotArchivePosition(cursor.archivePosition) + if (page.rows.length > 0) { + const last = page.rows.at(-1) as unknown as Record + if ( + cursor?.version !== 1 || + cursor.snapshotId !== checkpoint.snapshotId || + cursor.table !== table || + !Array.isArray(cursor.after) || + cursor.after.length !== keys[table].length || + cursor.after.some((value, index) => value !== last[keys[table][index]]) || + JSON.stringify(cursor) === JSON.stringify(checkpoint.cursor) + ) { + throw new WERR_INVALID_PARAMETER('page.cursor', 'the last row of this source view and table') + } + } else if (cursor !== undefined && JSON.stringify(cursor) !== JSON.stringify(checkpoint.cursor)) { + throw new WERR_INVALID_PARAMETER('page.cursor', 'the unchanged empty-page position') + } + // Entity merges and SQL serializers accept typed byte buffers. Keep the legacy + // public SyncChunk number[] declaration unchanged at this internal boundary. + const chunk = snapshotSyncPage( + { + identityKey: checkpoint.identityKey, + fromStorageIdentityKey: checkpoint.sourceStorageIdentityKey, + toStorageIdentityKey: checkpoint.destinationStorageIdentityKey, + offsets: [], + maxItems: 1000, + maxRoughSize: 16777216 + }, + { + userIdentityKey: checkpoint.identityKey, + fromStorageIdentityKey: checkpoint.sourceStorageIdentityKey, + toStorageIdentityKey: checkpoint.destinationStorageIdentityKey, + [table]: page.rows + } as SyncChunk + ).chunk + return { + chunk, + nextCursor: page.done ? null : JSON.stringify(cursor), + nextTable: checkpoint.tableIndex + Number(page.done) + } +} + +interface IdRow { + entity: keyof SyncMap + incomingId: number + localId: number +} +export interface SnapshotMapScope { + userId: number + sourceStorageIdentityKey: string +} + +interface PageReferences { + wanted: Map> + required: Map> + count: number +} + +function addReference(refs: PageReferences, entity: keyof SyncMap, id: unknown, parent: boolean): void { + if (!Number.isSafeInteger(id) || (id as number) < 1) throw new WERR_INVALID_PARAMETER('row', 'positive safe IDs') + const set = refs.wanted.get(entity) ?? new Set() + if (!set.has(id as number) && ++refs.count > 4096) + throw new SnapshotResourceLimitError('Snapshot page exceeds 4096 distinct ID references') + set.add(id as number) + refs.wanted.set(entity, set) + if (parent) { + const parents = refs.required.get(entity) ?? new Set() + parents.add(id as number) + refs.required.set(entity, parents) + } +} + +function collectNotifications(refs: PageReferences, row: Record): void { + const notify = JSON.parse(row.notify as string) as { transactionIds?: unknown[] } + if (notify.transactionIds == null) return + if (!Array.isArray(notify.transactionIds)) throw new WERR_INVALID_PARAMETER('notify', 'an array of transaction IDs') + // Global request notification lists may also contain other profiles. Only + // mappings established for this profile may survive the existing merge. + for (const id of notify.transactionIds) addReference(refs, 'transaction', id, false) +} + +function collectReferences(sourceUserId: number, table: SnapshotSyncTable, chunk: SyncChunk): PageReferences { + const refs: PageReferences = { wanted: new Map(), required: new Map(), count: 0 } + const rows = chunk[table] as unknown as Array> + const owned = !['provenTxs', 'provenTxReqs', 'txLabelMaps', 'outputTagMaps'].includes(table) + for (const row of rows) { + if ((owned || row.userId !== undefined) && row.userId !== sourceUserId) + throw new WERR_INVALID_OPERATION('Snapshot row belongs to another profile') + for (const [field, entity] of Object.entries(ids)) { + if (row[field] != null) addReference(refs, entity, row[field], entity !== entities[table]) + } + if (table === 'provenTxReqs') collectNotifications(refs, row) + } + return refs +} + +/** Produce one SQL parameter batch at a time without creating an eager query queue. */ +function* idBatches(values: T[]): Generator { + for (let offset = 0; offset < values.length; offset += 128) yield values.slice(offset, offset + 128) +} + +/** Load only incoming IDs and their parent references; never parse or overwrite the legacy JSON map. */ +export async function loadSnapshotIdMap( + k: Knex, + scope: SnapshotMapScope, + sourceUserId: number, + table: SnapshotSyncTable, + chunk: SyncChunk +): Promise<{ map: SyncMap; persist: () => Promise }> { + const { wanted, required } = collectReferences(sourceUserId, table, chunk) + const map = createSyncMap() + const known = new Map>() + await runInSeries(wanted, async ([entity, values]) => { + await runInSeries(idBatches([...values]), async batch => { + const found: IdRow[] = await k('snapshot_sync_ids') + .select('incomingId', 'localId') + .where({ ...scope, entity }) + .whereIn('incomingId', batch) + for (const row of found) map[entity].idMap[row.incomingId] = row.localId + }) + known.set(entity, new Set(Object.keys(map[entity].idMap).map(Number))) + }) + for (const [entity, values] of required) { + for (const id of values) { + if (map[entity].idMap[id] === undefined) + throw new WERR_INVALID_OPERATION('Snapshot parent mapping is missing; restart from the durable source view') + } + } + return { + map, + async persist() { + const additions: Array = [] + for (const [entity, values] of wanted) { + for (const id of values) { + const localId = map[entity].idMap[id] + if (localId !== undefined && !known.get(entity)?.has(id)) + additions.push({ ...scope, entity, incomingId: id, localId }) + } + } + await runInSeries(idBatches(additions), async batch => { + await k('snapshot_sync_ids').insert(batch) + }) + } + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncSession.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncSession.test.ts new file mode 100644 index 000000000..fa173d894 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/SnapshotSyncSession.test.ts @@ -0,0 +1,391 @@ +import { runSnapshotSyncSession } from './runSnapshotSyncSession' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import type { SnapshotSyncCheckpoint, SnapshotSyncCommit, SnapshotSyncStorage } from './SnapshotSync' +import type { SyncSessionOptions, SyncSessionProgress } from '../sync/syncSession' +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import { SyncPageBudget } from '../sync/SyncPageBudget' + +function session() { + const checkpoint: SnapshotSyncCheckpoint = { + version: 1, + sessionId: 'a'.repeat(64), + snapshotId: 'b'.repeat(64), + identityKey: 'identity', + sourceStorageIdentityKey: 'source', + destinationStorageIdentityKey: 'destination', + tableIndex: 11, + sequence: 11, + done: false, + cursor: { version: 1, snapshotId: 'b'.repeat(64), table: 'provenTxReqs', after: [7] } + } + const view = { + isOpen: true, + expiresAt: Date.now() + 10000, + closed: Promise.resolve(), + readPage: jest.fn(async () => ({ rows: [], payloadBytes: 0, done: true })) + } as unknown as WalletReadSnapshot + const apply = jest.fn(async (): Promise => ({ + checkpoint: { ...checkpoint, cursor: undefined, sequence: 12, tableIndex: 12, done: true }, + inserts: 2, + updates: 3 + })) + const destination = { + begin: jest.fn(async () => checkpoint), + prepare: jest.fn(async () => apply) + } as unknown as SnapshotSyncStorage + const ownership = { commit: async (work: () => Promise): Promise => await work() } + const commit = jest.spyOn(ownership, 'commit') + return { + input: { view, destination, commit: ownership.commit, activeStorage: 'source' }, + checkpoint, + view, + destination, + apply, + commit + } +} + +test.each([ + { maxItems: 0 }, + { maxItems: -1 }, + { maxItems: 1.5 }, + { maxItems: 1001 }, + { maxItems: NaN }, + { maxRoughSize: 0 }, + { maxRoughSize: -1 }, + { maxRoughSize: 1.5 }, + { maxRoughSize: 10000001 }, + { maxRoughSize: Infinity } +])('rejects invalid limits before opening destination state: %j', async options => { + const f = session() + await expect(runSnapshotSyncSession(f.input, options)).rejects.toThrow('limits') + expect(f.commit).not.toHaveBeenCalled() +}) + +test('progress uses detached checkpoints and reports the committed counts and timings', async () => { + const f = session() + Reflect.set(f.checkpoint.cursor!, 'archivePosition', { + version: 1, + archiveId: 'c'.repeat(64), + sequence: 12, + rowOffset: 7 + }) + const states: SyncSessionProgress['state'][] = [] + const result = await runSnapshotSyncSession(f.input, { + maxItems: 17, + maxRoughSize: 2048, + onProgress: progress => { + states.push(progress.state) + if (progress.snapshotCheckpoint?.cursor) Reflect.set(progress.snapshotCheckpoint.cursor.after, '0', 999) + if (progress.snapshotCheckpoint?.cursor?.archivePosition) + Reflect.set(progress.snapshotCheckpoint.cursor.archivePosition, 'rowOffset', 999) + if (progress.snapshotCheckpoint) progress.snapshotCheckpoint.identityKey = 'changed-by-listener' + if (progress.state === 'committed') { + expect(progress).toMatchObject({ pages: 1, inserts: 2, updates: 3 }) + for (const value of [progress.readMs, progress.prepareMs, progress.commitMs, progress.queueMs]) + expect(value).toBeGreaterThanOrEqual(0) + } + } + }) + expect(states).toEqual(['reading', 'preparing', 'committing', 'committed', 'completed']) + expect(f.view.readPage).toHaveBeenCalledWith('provenTxReqs', f.checkpoint.cursor, { + maxRows: expect.any(Number), + maxBytes: 2048 + }) + const limits = (f.view.readPage as jest.Mock).mock.calls[0][2] + expect(limits.maxRows).toBeGreaterThanOrEqual(1) + expect(limits.maxRows).toBeLessThanOrEqual(17) + expect(f.checkpoint.cursor!.after).toEqual([7]) + expect(f.checkpoint.cursor!.archivePosition!.rowOffset).toBe(7) + expect(result).toMatchObject({ + status: 'completed', + mode: 'paged', + pages: 1, + inserts: 2, + updates: 3, + snapshotCheckpoint: { done: true, identityKey: 'identity' } + }) + expect(f.commit).toHaveBeenCalledTimes(2) +}) + +test.each([ + 'before-start', + 'queued-start', + 'after-start', + 'reading', + 'after-read', + 'preparing', + 'after-prepare', + 'committing', + 'committed' +])('cancellation at %s reports the durable outcome and stops the next write', async boundary => { + const f = session() + const controller = new AbortController() + const states: string[] = [] + if (boundary === 'before-start') controller.abort() + if (boundary === 'queued-start') + f.commit.mockImplementationOnce(async work => { + controller.abort() + return await work() + }) + if (boundary === 'after-start') + (f.destination.begin as jest.Mock).mockImplementationOnce(async () => { + controller.abort() + return f.checkpoint + }) + if (boundary === 'after-read') + (f.view.readPage as jest.Mock).mockImplementationOnce(async () => { + controller.abort() + return { rows: [], payloadBytes: 0, done: true } + }) + if (boundary === 'after-prepare') + (f.destination.prepare as jest.Mock).mockImplementationOnce(async () => { + controller.abort() + return f.apply + }) + const result = await runSnapshotSyncSession(f.input, { + signal: controller.signal, + onProgress: progress => { + states.push(progress.state) + if (progress.state === boundary) controller.abort() + } + }) + const committed = boundary === 'committed' + expect(result.status).toBe('cancelled') + expect(result.pages).toBe(Number(committed)) + expect(f.apply).toHaveBeenCalledTimes(Number(committed)) + expect(states.slice(-2)).toEqual(['cancelling', 'cancelled']) + expect(states).not.toContain('completed') + expect(f.destination.begin).toHaveBeenCalledTimes(Number(!['before-start', 'queued-start'].includes(boundary))) + expect(f.view.readPage).toHaveBeenCalledTimes( + Number(['after-read', 'preparing', 'after-prepare', 'committing', 'committed'].includes(boundary)) + ) + expect(f.destination.prepare).toHaveBeenCalledTimes( + Number(['after-prepare', 'committing', 'committed'].includes(boundary)) + ) + const stages = ['reading', 'preparing', 'committing', 'committed'] + const stageCount = { + 'before-start': 0, + 'queued-start': 0, + 'after-start': 0, + reading: 1, + 'after-read': 1, + preparing: 2, + 'after-prepare': 2, + committing: 3, + committed: 4 + }[boundary]! + expect(states).toEqual([...stages.slice(0, stageCount), 'cancelling', 'cancelled']) +}) + +test.each([ + ['version', 2], + ['snapshotId', 'another-view'], + ['table', 'txLabels'], + ['after', [9]] +])('a changed acknowledged cursor %s cannot become the next read position', async (field, value) => { + const f = session() + const cursor = { version: 1 as const, snapshotId: 'b'.repeat(64), table: 'provenTxReqs' as const, after: [8] } + ;(f.view.readPage as jest.Mock).mockResolvedValue({ + rows: [{ provenTxReqId: 8 }], + payloadBytes: 128, + done: false, + cursor + }) + f.apply.mockResolvedValue({ + inserts: 1, + updates: 0, + checkpoint: { ...f.checkpoint, sequence: 12, cursor: { ...cursor, [field as string]: value } } + }) + await expect(runSnapshotSyncSession(f.input, {})).rejects.toThrow('acknowledgement does not match') + expect(f.view.readPage).toHaveBeenCalledTimes(1) +}) + +test('a nonterminal acknowledgement advances a detached cursor before finishing its table', async () => { + const f = session() + const cursor: WalletSnapshotCursor = { + version: 1, + snapshotId: 'b'.repeat(64), + table: 'provenTxReqs', + after: [8], + archivePosition: { version: 1, archiveId: 'c'.repeat(64), sequence: 12, rowOffset: 8 } + } + ;(f.view.readPage as jest.Mock).mockResolvedValueOnce({ + rows: [{ provenTxReqId: 8 }], + payloadBytes: 128, + done: false, + cursor + }) + const acknowledged: SnapshotSyncCheckpoint = { + ...f.checkpoint, + sequence: 12, + cursor: { ...cursor, after: [8], archivePosition: { ...cursor.archivePosition! } } + } + f.apply.mockResolvedValueOnce({ inserts: 1, updates: 0, checkpoint: acknowledged }) + f.apply.mockResolvedValueOnce({ + inserts: 0, + updates: 0, + checkpoint: { ...f.checkpoint, sequence: 13, tableIndex: 12, done: true, cursor: undefined } + }) + const result = await runSnapshotSyncSession(f.input, { + onProgress: progress => { + if (progress.state === 'committed' && progress.pages === 1) { + Reflect.set(acknowledged.cursor!.after, '0', 999) + Reflect.set(acknowledged.cursor!.archivePosition!, 'rowOffset', 999) + acknowledged.sequence = 999 + } + } + }) + expect((f.view.readPage as jest.Mock).mock.calls[1][1]).toEqual(cursor) + expect(result).toMatchObject({ status: 'completed', pages: 2, inserts: 1, updates: 0 }) +}) + +test.each(['omitted', 'archive', 'sequence', 'offset'] as const)( + 'an acknowledgement with %s archive position cannot become the next read position', + async change => { + const f = session() + const position = { version: 1 as const, archiveId: 'c'.repeat(64), sequence: 12, rowOffset: 8 } + const cursor: WalletSnapshotCursor = { + version: 1, + snapshotId: 'b'.repeat(64), + table: 'provenTxReqs', + after: [8], + archivePosition: position + } + ;(f.view.readPage as jest.Mock).mockResolvedValue({ + rows: [{ provenTxReqId: 8 }], + payloadBytes: 128, + done: false, + cursor + }) + const changed = { ...position } + if (change === 'archive') changed.archiveId = 'd'.repeat(64) + if (change === 'sequence') changed.sequence++ + if (change === 'offset') changed.rowOffset++ + f.apply.mockResolvedValue({ + inserts: 1, + updates: 0, + checkpoint: { + ...f.checkpoint, + sequence: 12, + cursor: { ...cursor, archivePosition: change === 'omitted' ? undefined : changed } + } + }) + await expect(runSnapshotSyncSession(f.input, {})).rejects.toThrow('acknowledgement does not match') + expect(f.view.readPage).toHaveBeenCalledTimes(1) + } +) + +test.each(['closed', 'expired', 'cleanup-failed'] as const)( + 'source %s while queued prevents a destination write', + async outcome => { + const f = session() + const cleanup = new Error('physical cleanup failed') + const options: SyncSessionOptions = { + onProgress: progress => { + if (progress.state !== 'committing') return + Object.defineProperty(f.view, 'isOpen', { value: false }) + if (outcome === 'expired') Object.defineProperty(f.view, 'expiresAt', { value: 0 }) + if (outcome === 'cleanup-failed') { + const rejected = Promise.reject(cleanup) + void rejected.catch(() => undefined) + Object.defineProperty(f.view, 'closed', { value: rejected }) + } + } + } + const running = runSnapshotSyncSession(f.input, options) + if (outcome === 'expired') await expect(running).rejects.toBeInstanceOf(SnapshotResourceLimitError) + else if (outcome === 'cleanup-failed') await expect(running).rejects.toBe(cleanup) + else await expect(running).rejects.toThrow('source closed') + expect(f.apply).not.toHaveBeenCalled() + } +) + +test('an already completed destination session does not read or prepare another page', async () => { + const f = session() + f.checkpoint.tableIndex = 12 + f.checkpoint.done = true + const result = await runSnapshotSyncSession(f.input, {}) + expect(result).toMatchObject({ status: 'completed', pages: 0, snapshotCheckpoint: { done: true } }) + expect(f.view.readPage).not.toHaveBeenCalled() + expect(f.destination.prepare).not.toHaveBeenCalled() +}) + +test.each([ + ['version', 2], + ['sessionId', 'changed'], + ['identityKey', 'another-profile'], + ['sourceStorageIdentityKey', 'another-source'], + ['destinationStorageIdentityKey', 'another-destination'], + ['snapshotId', 'another-view'], + ['sequence', 11], + ['tableIndex', 11], + ['done', false], + ['cursor', { version: 1, snapshotId: 'b'.repeat(64), table: 'provenTxReqs', after: [7] }] +])('a mismatched acknowledged %s rejects before counting or reading another page', async (field, value) => { + const f = session() + const commit = await f.apply() + f.apply.mockClear() + f.apply.mockResolvedValue({ ...commit, checkpoint: { ...commit.checkpoint, [field as string]: value } }) + const states: string[] = [] + await expect( + runSnapshotSyncSession(f.input, { onProgress: progress => states.push(progress.state) }) + ).rejects.toThrow('acknowledgement does not match') + expect(f.apply).toHaveBeenCalledTimes(1) + expect(f.view.readPage).toHaveBeenCalledTimes(1) + expect(states).not.toContain('committed') + expect(states).not.toContain('completed') +}) + +test('progress timings separate read, preparation, queue and commit costs', async () => { + const f = session() + let now = 1000 + const clock = jest.spyOn(Date, 'now').mockImplementation(() => now) + const budget = jest.spyOn(SyncPageBudget.prototype, 'committed') + const progress: SyncSessionProgress[] = [] + ;(f.view.readPage as jest.Mock).mockImplementation(async () => { + now += 11 + return { rows: [], payloadBytes: 0, done: true } + }) + ;(f.destination.prepare as jest.Mock).mockImplementation(async () => { + now += 13 + return f.apply + }) + const apply = f.apply.getMockImplementation()! + f.apply.mockImplementation(async () => { + now += 19 + return await apply() + }) + let commits = 0 + f.commit.mockImplementation(async work => { + if (++commits === 2) now += 17 + return await work() + }) + try { + await runSnapshotSyncSession(f.input, { onProgress: event => progress.push(event) }) + expect(progress.find(event => event.state === 'preparing')).toMatchObject({ readMs: 11 }) + expect(progress.find(event => event.state === 'committing')).toMatchObject({ readMs: 11, prepareMs: 13 }) + expect(progress.find(event => event.state === 'committed')).toMatchObject({ + readMs: 11, + prepareMs: 13, + queueMs: 17, + commitMs: 19 + }) + expect(budget).toHaveBeenCalledWith(expect.any(Object), 43, 24) + } finally { + budget.mockRestore() + clock.mockRestore() + } +}) + +test.each([1, 10000000])('the exact byte ceiling %s remains accepted', async maxRoughSize => { + const f = session() + expect(await runSnapshotSyncSession(f.input, { maxItems: 1, maxRoughSize })).toMatchObject({ + status: 'completed', + pages: 1 + }) + expect(f.view.readPage).toHaveBeenCalledWith('provenTxReqs', f.checkpoint.cursor, { + maxRows: 1, + maxBytes: maxRoughSize + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/StorageKnex.retainedSnapshot.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/StorageKnex.retainedSnapshot.test.ts new file mode 100644 index 000000000..f17c0380d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/StorageKnex.retainedSnapshot.test.ts @@ -0,0 +1,253 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { knex } from 'knex' +import { StorageKnex } from '../StorageKnex' +import { StorageIdb } from '../StorageIdb' +import { StorageProvider } from '../StorageProvider' +import { runInSeries } from '../../utility/runInSeries' + +const stores: StorageKnex[] = [] +const directories: string[] = [] +const identity = '02' + '11'.repeat(32) + +async function pair(): Promise<[StorageKnex, StorageKnex]> { + const directory = await mkdtemp(join(tmpdir(), 'retained-wallet-view-')) + directories.push(directory) + const open = () => { + const store = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 1000 + }) + }) + stores.push(store) + return store + } + const source = open() + await source.knex.raw('PRAGMA journal_mode = WAL') + await source.migrate('retained source', 'source-storage') + await source.makeAvailable() + const writer = open() + await writer.makeAvailable() + return [source, writer] +} + +afterEach(async () => { + jest.restoreAllMocks() + await runInSeries(stores.splice(0), store => store.destroy()) + await runInSeries(directories.splice(0), directory => rm(directory, { recursive: true, force: true })) +}) + +test('retains an already-pinned SQLite view across independent writes, idle periods and repeated keyset reads', async () => { + const [source, writer] = await pair() + const { user } = await source.findOrInsertUser(identity) + const when = new Date('2026-01-01T00:00:00.000Z') + await runInSeries( + Array.from({ length: 12 }, (_, index) => index), + async index => { + await source.insertTxLabel({ + txLabelId: 0, + userId: user.userId, + label: `label ${index}`, + isDeleted: index === 5, + created_at: when, + updated_at: when + }) + } + ) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await source.findOrInsertTxLabel(other.userId, 'foreign label') + const expected = await source.findTxLabels({ partial: { userId: user.userId } }) + expect(source.supportsRetainedReadSnapshot()).toBe(true) + const view = await source.openReadSnapshot() + // There has been no consumer read yet. Opening must already have pinned the + // view, rather than choosing an unrelated point in time on the first page. + await writer.updateTxLabel(expected[0].txLabelId, { label: 'changed after open', updated_at: when }) + await writer.updateTxLabel(expected[7].txLabelId, { isDeleted: true, updated_at: when }) + await writer.findOrInsertTxLabel(user.userId, 'inserted after open') + const page = (after: number) => + view.read(trx => + source + .toDb(trx)<{ + userId: number + txLabelId: number + label: string + isDeleted: boolean | number + }>('tx_labels') + .select('txLabelId', 'label', 'isDeleted') + .where({ userId: user.userId }) + .where('txLabelId', '>', after) + .orderBy('txLabelId') + .limit(4) + ) + const first = await page(0) + await new Promise(resolve => setImmediate(resolve)) + const second = await page(first[3].txLabelId) + const third = await page(second[3].txLabelId) + expect(await page(third[3].txLabelId)).toEqual([]) + expect(await page(0)).toEqual(first) + const result = [...first, ...second, ...third] + expect(result.map(row => [row.txLabelId, row.label, Boolean(row.isDeleted)])).toEqual( + expected.map(row => [row.txLabelId, row.label, row.isDeleted]) + ) + await view.close() + const next = await source.openReadSnapshot() + const current = await next.read(trx => source.findTxLabels({ partial: { userId: user.userId }, trx })) + expect(current).toHaveLength(13) + expect(current.find(row => row.txLabelId === expected[0].txLabelId)?.label).toBe('changed after open') + expect(current.find(row => row.txLabelId === expected[7].txLabelId)?.isDeleted).toBe(true) + await next.close() +}) + +test('cancellation keeps the provider slot occupied until an in-flight database read drains', async () => { + const [source] = await pair() + const { user } = await source.findOrInsertUser(identity) + const controller = new AbortController() + const view = await source.openReadSnapshot({ signal: controller.signal }) + let release!: () => void + const held = new Promise(resolve => { + release = resolve + }) + let started!: () => void + const reading = new Promise(resolve => { + started = resolve + }) + const pending = view.read(async trx => { + const rows = await source.findTxLabels({ partial: { userId: user.userId }, trx }) + started() + await held + return rows + }) + const outcome = pending.catch(error => error) + try { + await reading + controller.abort() + await expect(source.openReadSnapshot()).rejects.toThrow('already has a retained read snapshot') + release() + expect(await outcome).toBeInstanceOf(Error) + await view.closed + const fresh = await source.openReadSnapshot() + await fresh.close() + } finally { + release() + await view.close() + } +}) + +test('a failed read rolls back and permits a later retained view and ordinary write', async () => { + const [source] = await pair() + const { user } = await source.findOrInsertUser(identity) + const view = await source.openReadSnapshot() + const failure = new Error('synthetic snapshot query failure') + await expect( + view.read(async trx => { + await source.findUsers({ partial: { userId: user.userId }, trx }) + throw failure + }) + ).rejects.toBe(failure) + await expect(view.closed).rejects.toBe(failure) + await source.findOrInsertTxLabel(user.userId, 'after rollback') + const fresh = await source.openReadSnapshot() + expect(await fresh.read(trx => source.findTxLabels({ partial: { userId: user.userId }, trx }))).toHaveLength(1) + await fresh.close() +}) + +test('provider destruction closes its idle retained view before destroying the pool', async () => { + const [source] = await pair() + const view = await source.openReadSnapshot() + await source.destroy() + stores.splice(stores.indexOf(source), 1) + await view.closed + expect(view.isOpen).toBe(false) + await expect(view.read(async () => 1)).rejects.toThrow('closed') +}) + +test('IndexedDB retains its scoped snapshot support but explicitly refuses an idle retained transaction', async () => { + const source = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + expect(source.supportsReadSnapshot()).toBe(true) + expect(source.getSnapshotSync()).toBeUndefined() + expect(source.supportsRetainedReadSnapshot()).toBe(false) + await expect(source.openReadSnapshot()).rejects.toThrow('Retained read snapshots are not supported') +}) + +test('reserves the provider slot before acquisition hooks can re-enter opening', async () => { + const [source] = await pair() + const aborted = new AbortController() + aborted.abort() + const run = source.readSnapshot.bind(source) + let nested: Promise | undefined + jest.spyOn(source, 'readSnapshot').mockImplementationOnce(async read => { + nested = source.openReadSnapshot({ signal: aborted.signal }).catch(error => error) + return await run(read) + }) + const view = await source.openReadSnapshot() + expect(await nested).toEqual( + expect.objectContaining({ message: expect.stringContaining('already has a retained read snapshot') }) + ) + await view.close() +}) + +test('provider destruction still destroys the pool if retained-view cleanup reports failure', async () => { + const [source] = await pair() + const cleanupFailure = new Error('synthetic cleanup failure') + const run = source.readSnapshot.bind(source) + jest.spyOn(source, 'readSnapshot').mockImplementationOnce(async read => { + await run(read) + throw cleanupFailure + }) + const view = await source.openReadSnapshot() + await expect(source.destroy()).rejects.toBe(cleanupFailure) + stores.splice(stores.indexOf(source), 1) + await expect(view.closed).rejects.toBe(cleanupFailure) + await expect(source.knex.raw('SELECT 1')).rejects.toThrow('Unable to acquire a connection') +}) + +test('SQL providers with an unsupported database retain the explicit unsupported fallback', async () => { + const [source] = await pair() + jest.spyOn(source, 'supportsReadSnapshot').mockReturnValue(false) + const read = jest.spyOn(source, 'readSnapshot') + expect(source.supportsRetainedReadSnapshot()).toBe(false) + await expect(source.openReadSnapshot()).rejects.toThrow('Retained read snapshots are not supported') + expect(read).not.toHaveBeenCalled() +}) + +test.each([false, true])( + 'destruction fences retained-view admission before asynchronous cleanup (existing view: %s)', + async existing => { + const [source] = await pair() + if (existing) await source.openReadSnapshot() + let release!: () => void + const cleanup = new Promise(resolve => { + release = resolve + }) + let entered!: () => void + const stopping = new Promise(resolve => { + entered = resolve + }) + jest.spyOn(source, 'stopPreparedBeefTasks').mockImplementationOnce(async () => { + entered() + await cleanup + }) + const destruction = source.destroy() + try { + await stopping + // Observe/close an incorrectly admitted view so a failed assertion cannot + // strand the pool. The provider must refuse before any acquisition. + const read = jest.spyOn(source, 'readSnapshot') + const result = await source.openReadSnapshot().catch(error => error) + if (!(result instanceof Error)) await result.close() + expect(result).toEqual(expect.objectContaining({ message: expect.stringContaining('destruction begins') })) + expect(read).not.toHaveBeenCalled() + } finally { + release() + await destruction + stores.splice(stores.indexOf(source), 1) + } + await expect(source.openReadSnapshot()).rejects.toThrow('destruction begins') + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/WalletReadSnapshot.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/WalletReadSnapshot.ts new file mode 100644 index 000000000..fa92c60bc --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/WalletReadSnapshot.ts @@ -0,0 +1,87 @@ +import type * as tables from '../schema/tables' +import type { RetainedReadSnapshotOptions } from './RetainedReadSnapshot' + +/** Version-one retained row contract. Binary columns never expand into number arrays. */ +export type PackedSnapshotRow = { [K in keyof T]: PackedSnapshotValue } +type PackedSnapshotValue = T extends number[] ? Uint8Array : T + +export interface WalletSnapshotTables { + provenTxs: tables.TableProvenTx + provenTxReqs: tables.TableProvenTxReq + outputBaskets: tables.TableOutputBasket + transactions: tables.TableTransaction + commissions: tables.TableCommission + outputs: tables.TableOutput + outputTags: tables.TableOutputTag + outputTagMaps: tables.TableOutputTagMap + txLabels: tables.TableTxLabel + txLabelMaps: tables.TableTxLabelMap + certificates: tables.TableCertificate + certificateFields: tables.TableCertificateField + syncStates: tables.TableSyncState +} + +export type WalletSnapshotTable = keyof WalletSnapshotTables + +/** Verified position inside an immutable remote archive; never an authorization credential. */ +export interface WalletSnapshotArchivePosition { + readonly version: 1 + readonly archiveId: string + readonly sequence: number + readonly rowOffset: number +} + +/** A position within this retained view, not a durable checkpoint or authorization credential. */ +export interface WalletSnapshotCursor { + readonly version: 1 + readonly snapshotId: string + readonly table: WalletSnapshotTable + readonly after: ReadonlyArray + readonly archivePosition?: WalletSnapshotArchivePosition +} + +export interface WalletSnapshotPageLimits { + /** Defaults to 128. Integer from 1 to 1,000. */ + maxRows?: number + /** + * Defaults to 262,144. Integer from 1 to 16,777,216. The SQL preflight charges + * twice each cell's stored byte length plus 64 bytes per cell before fetching + * payloads. This is a payload allocation budget, not encoded wire size or RSS. + * An individual row over the budget rejects; it is never silently omitted. + */ + maxBytes?: number +} + +export interface WalletSnapshotPage { + rows: Array> + /** Preflight charge for the returned rows. */ + payloadBytes: number + /** True only when this table's retained view has been exhausted. */ + done: boolean + /** Last returned position, usable for retry or continuation only in this view. */ + cursor?: WalletSnapshotCursor +} + +export interface WalletReadSnapshot { + readonly version: 1 + readonly snapshotId: string + readonly sourceStorage: tables.TableSettings + readonly user: tables.TableUser + readonly expiresAt: number + readonly isOpen: boolean + readonly closed: Promise + /** + * One read at a time. Repeating a cursor and limits repeats the same page. + * Rows include tombstones and stored values; no operational policy is applied. + * This local interface is not exposed over RPC and does not validate a complete + * portable document. Settings, user and every page share the same read view. + */ + readPage: ( + table: T, + cursor?: WalletSnapshotCursor, + limits?: WalletSnapshotPageLimits + ) => Promise> + close: () => Promise +} + +export type WalletReadSnapshotOptions = RetainedReadSnapshotOptions diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveCapture.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveCapture.test.ts new file mode 100644 index 000000000..15c5c9f5c --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveCapture.test.ts @@ -0,0 +1,440 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { SNAPSHOT_SQLITE_INDEX_MIGRATION } from '../../schema/KnexMigrations' +import { decodeSyncTransfer } from '../../remoting/SyncTransfer' +import * as Transfer from '../../remoting/SyncTransfer' +import * as ArchiveSource from './KnexSnapshotArchiveSource' +import { runInNewContext } from 'node:vm' +import { seedArchiveClosure } from '../../../../test/utils/snapshotArchiveFixtures' +import { openKnexSnapshotArchiveSource } from './KnexSnapshotArchiveSource' +import { assertKnexSnapshotArchiveClosure } from './KnexSnapshotArchiveClosure' +import { captureKnexSnapshotArchive } from './captureKnexSnapshotArchive' +import { KnexSnapshotArchiveStore, snapshotArchiveTables } from './KnexSnapshotArchiveStore' +import { verifySnapshotArchiveDirectory, verifySnapshotArchivePage } from './SnapshotArchiveDirectory' + +const identity = '02' + '11'.repeat(32) +const foreign = '03' + '22'.repeat(32) +const stores: StorageKnex[] = [] +const directories: string[] = [] + +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'wallet-archive-capture-')) + directories.push(directory) + const open = () => { + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 1000 + }) + }) + stores.push(storage) + return storage + } + const writer = open() + await writer.knex.raw('PRAGMA journal_mode = WAL') + await writer.migrate('original source', 'original-source') + await writer.makeAvailable() + const { user } = await writer.findOrInsertUser(identity) + const { user: other } = await writer.findOrInsertUser(foreign) + await seedArchiveClosure(writer, user.userId, other.userId) + await writer.knex('outputs').where({ outputId: 1 }).update({ spentBy: 3 }) + const reader = open() + await reader.makeAvailable() + return { reader, writer, userId: user.userId, otherId: other.userId } +} + +afterEach(async () => { + jest.restoreAllMocks() + await Promise.all(stores.splice(0).map(storage => storage.destroy())) + await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))) +}) + +async function changeReference(writer: StorageKnex, table: string, key: string, field: string, value: number) { + if (table === 'commissions') await writer.knex('commissions').where({ transactionId: value }).delete() + const query = writer.knex(table).where({ [key]: 1 }) + if (table.endsWith('_map')) void query.where({ [field]: 1 }) + if (table === 'certificate_fields') { + await query.where({ certificateId: 1, fieldName: 'a' }).update({ [field]: value, fieldName: 'changed' }) + } else await query.update({ [field]: value }) +} + +test('captures all thirteen tables with original metadata, packed bytes and profile-bound relations', async () => { + const { reader, writer, userId } = await fixture() + const progress: Array<{ pages: number; rows: number; bytes: number }> = [] + const manifest = await captureKnexSnapshotArchive(reader, writer.knex, identity, 'test', { + onProgress: p => progress.push(p) + }) + expect(manifest.binding.sourceSchema).toBe(SNAPSHOT_SQLITE_INDEX_MIGRATION) + expect(manifest.binding.sourceStorage.storageName).toBe('original source') + expect(manifest.binding.sourceStorage.storageIdentityKey).toBe('original-source') + expect(manifest.binding.user).toMatchObject({ userId, identityKey: identity }) + expect(manifest.pages).toBe(13) + const store = new KnexSnapshotArchiveStore(writer.knex) + const verified = verifySnapshotArchiveDirectory(await store.directory(identity, manifest.archiveId), { + identityKey: identity, + chain: 'test', + sourceStorageIdentityKey: 'original-source', + archiveId: manifest.archiveId, + digest: manifest.digest, + sourceSchema: SNAPSHOT_SQLITE_INDEX_MIGRATION + }) + expect(verified.manifest).toEqual(manifest) + const captured: Record>> = {} + for (const [sequence, table] of snapshotArchiveTables.entries()) { + const page = await store.read(identity, manifest.archiveId, sequence) + const frame = decodeSyncTransfer(verifySnapshotArchivePage(page, verified.receipts[sequence])) as { + version: number + table: string + rows: Array> + } + expect(frame).toMatchObject({ version: 1, table }) + expect(frame.rows).toHaveLength(page.rows) + expect(page.done).toBe(true) + captured[table] = frame.rows + for (const row of frame.rows) { + if ('userId' in row) expect(row.userId).toBe(userId) + expect(row.created_at).toBe('2026-01-01T00:00:00.000Z') + } + } + expect(captured.provenTxs.map(row => row.provenTxId)).toEqual([1, 3]) + expect(captured.provenTxs[0].rawTx).toEqual(new Uint8Array([1, 1, 255])) + expect(captured.provenTxReqs.map(row => row.provenTxReqId)).toEqual([1, 3]) + expect(captured.transactions.map(row => row.transactionId)).toEqual([1, 3]) + expect(captured.txLabelMaps.map(row => [row.txLabelId, row.transactionId])).toEqual([ + [1, 1], + [1, 3], + [3, 3] + ]) + expect(captured.syncStates.map(row => [row.syncStateId, row.syncMap])).toEqual([ + [1, '{}'], + [3, '{}'] + ]) + expect(captured.certificateFields).toHaveLength(8) + expect(progress.at(-1)).toMatchObject({ pages: 13, rows: manifest.rows }) + expect(progress.at(-1)!.bytes).toBeGreaterThan(0) + expect(progress[0].pages).toBe(1) + const next = await reader.openReadSnapshot() + await next.close() + await store.close(identity, manifest.archiveId) +}) + +test('preserves nullable proof, basket and spending references in an incomplete wallet history', async () => { + const { reader, writer } = await fixture() + await writer.knex('transactions').where({ transactionId: 1 }).update({ provenTxId: null }) + await writer.knex('proven_tx_reqs').where({ provenTxReqId: 1 }).update({ provenTxId: null }) + await writer.knex('outputs').where({ outputId: 1 }).update({ basketId: null, spentBy: null }) + const manifest = await captureKnexSnapshotArchive(reader, writer.knex, identity, 'test') + const store = new KnexSnapshotArchiveStore(writer.knex) + for (const [sequence, field] of [ + [1, 'provenTxId'], + [3, 'provenTxId'], + [5, 'basketId'] + ] as const) { + const page = await store.read(identity, manifest.archiveId, sequence) + const frame = decodeSyncTransfer(page.bytes) as { rows: Array> } + expect(frame.rows[0][field]).toBeUndefined() + if (sequence === 5) expect(frame.rows[0].spentBy).toBeUndefined() + } + await store.close(identity, manifest.archiveId) +}) + +test('source schema, primary history and closure stay pinned while an independent writer changes them', async () => { + const { reader, writer } = await fixture() + const source = await openKnexSnapshotArchiveSource(reader, identity) + const originalPrimary = source.user.activeStorage + await writer.knex('users').where({ identityKey: identity }).update({ activeStorage: 'replacement' }) + await writer.knex('knex_migrations').insert({ name: 'future-schema', batch: 99, migration_time: new Date() }) + await writer.knex('outputs').where({ outputId: 1 }).update({ basketId: 2 }) + expect(source.sourceSchema).toBe(SNAPSHOT_SQLITE_INDEX_MIGRATION) + expect(source.user.activeStorage).toBe(originalPrimary) + await expect(source.validateClosure()).resolves.toBeUndefined() + expect((await source.readPage('outputs')).rows[0].basketId).toBe(1) + await source.close() + const changed = await openKnexSnapshotArchiveSource(reader, identity) + expect(changed.sourceSchema).toBe('future-schema') + expect(changed.user.activeStorage).toBe('replacement') + await expect(changed.validateClosure()).rejects.toThrow('relation') + await expect(changed.closed).rejects.toThrow('relation') +}) + +test.each([ + ['transactions', 'transactionId', 'provenTxId'], + ['proven_tx_reqs', 'provenTxReqId', 'provenTxId'], + ['commissions', 'commissionId', 'transactionId'], + ['outputs', 'outputId', 'transactionId'], + ['outputs', 'outputId', 'basketId'], + ['outputs', 'outputId', 'spentBy'], + ['tx_labels_map', 'txLabelId', 'transactionId'], + ['tx_labels_map', 'transactionId', 'txLabelId'], + ['output_tags_map', 'outputTagId', 'outputId'], + ['output_tags_map', 'outputId', 'outputTagId'], + ['certificate_fields', 'certificateId', 'userId'], + ['certificate_fields', 'userId', 'certificateId'] +])('refuses dangling %s.%s/%s without sealing or leaking a reservation', async (table, key, field) => { + const { reader, writer } = await fixture() + await writer.knex.raw('PRAGMA foreign_keys = OFF') + await changeReference(writer, table, key, field, 999) + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test')).rejects.toThrow('relation') + expect(await writer.knex('snapshot_archives')).toHaveLength(0) + expect(await writer.knex('snapshot_archive_pages')).toHaveLength(0) + expect(await writer.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test.each([ + ['commissions', 'commissionId', 'transactionId'], + ['outputs', 'outputId', 'transactionId'], + ['outputs', 'outputId', 'basketId'], + ['outputs', 'outputId', 'spentBy'], + ['tx_labels_map', 'txLabelId', 'transactionId'], + ['tx_labels_map', 'transactionId', 'txLabelId'], + ['output_tags_map', 'outputTagId', 'outputId'], + ['output_tags_map', 'outputId', 'outputTagId'], + ['certificate_fields', 'certificateId', 'userId'], + ['certificate_fields', 'userId', 'certificateId'] +])('refuses existing foreign-profile targets in %s.%s/%s', async (table, key, field) => { + const { reader, writer } = await fixture() + await changeReference(writer, table, key, field, 2) + const source = await openKnexSnapshotArchiveSource(reader, identity) + await expect(source.validateClosure()).rejects.toThrow('relation') + await expect(source.closed).rejects.toThrow('relation') +}) + +test('cancellation and progress callback failures discard committed staging and release the reader', async () => { + const { reader, writer } = await fixture() + const signal = new AbortController() + await expect( + captureKnexSnapshotArchive(reader, writer.knex, identity, 'test', { + signal: signal.signal, + onProgress: () => signal.abort() + }) + ).rejects.toThrow('cancelled') + expect(await writer.knex('snapshot_archive_pages')).toHaveLength(0) + const failure = new Error('synthetic progress failure') + await expect( + captureKnexSnapshotArchive(reader, writer.knex, identity, 'test', { + onProgress: () => { + throw failure + } + }) + ).rejects.toBe(failure) + expect(await writer.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + const next = await reader.openReadSnapshot() + await next.close() +}) + +test('invalid binding, network, shared-pool and already-cancelled requests fail with no partial archive', async () => { + const { reader, writer } = await fixture() + await expect(captureKnexSnapshotArchive(reader, reader.knex, identity, 'test')).rejects.toThrow('separate') + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'mock')).rejects.toThrow('chain') + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'main')).rejects.toThrow('chain') + await expect(captureKnexSnapshotArchive(reader, writer.knex, 'invalid', 'test')).rejects.toThrow('identityKey') + await expect(captureKnexSnapshotArchive(reader, writer.knex, '02' + '33'.repeat(32), 'test')).rejects.toThrow( + 'existing wallet profile' + ) + const signal = new AbortController() + signal.abort() + await expect( + captureKnexSnapshotArchive(reader, writer.knex, identity, 'test', { signal: signal.signal }) + ).rejects.toThrow('cancelled') + expect(await writer.knex('snapshot_archives')).toHaveLength(0) + const queries: string[] = [] + writer.knex.on('query', query => queries.push(query.sql)) + for (const id of [0, -1, 1.5, NaN, Number.MAX_SAFE_INTEGER + 1]) + await expect(assertKnexSnapshotArchiveClosure(writer.knex, id)).rejects.toThrow('userId') + expect(queries).toHaveLength(0) +}) + +test('malformed identities refuse the source before acquiring a database read view', async () => { + const { reader } = await fixture() + const queries: string[] = [] + reader.knex.on('query', query => queries.push(query.sql)) + for (const key of [null, 42, '', 'invalid', 'prefix' + identity, identity + 'suffix', '04' + '11'.repeat(32)]) { + await expect(openKnexSnapshotArchiveSource(reader, key as string)).rejects.toThrow('compressed public identity key') + } + expect(queries).toHaveLength(0) +}) + +test('capture checks source identity independently and forwards cancellation to the retained view', async () => { + const { reader, writer } = await fixture() + const source = await openKnexSnapshotArchiveSource(reader, identity) + jest.spyOn(ArchiveSource, 'openKnexSnapshotArchiveSource').mockResolvedValue({ + ...source, + user: { ...source.user, identityKey: foreign } + }) + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test')).rejects.toThrow('requested profile') + expect(source.isOpen).toBe(false) + expect(await writer.knex('snapshot_archives')).toHaveLength(0) + jest.restoreAllMocks() + const signal = new AbortController() + const open = reader.openReadSnapshot.bind(reader) + let captured: Awaited> | undefined + jest.spyOn(reader, 'openReadSnapshot').mockImplementation(async options => { + captured = await open(options) + return captured + }) + await expect( + captureKnexSnapshotArchive(reader, writer.knex, identity, 'test', { + signal: signal.signal, + lifetimeMs: 60000, + onProgress: () => { + signal.abort() + expect(captured?.isOpen).toBe(false) + } + }) + ).rejects.toThrow('cancelled') + expect(await writer.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('missing or invalid migration metadata refuses opening and releases read capacity', async () => { + const { reader, writer } = await fixture() + const metadata = await writer.knex('knex_migrations').orderBy('id', 'desc').first() + for (const name of ['', 'x'.repeat(257)]) { + await writer.knex('knex_migrations').where({ id: metadata.id }).update({ name }) + await expect(openKnexSnapshotArchiveSource(reader, identity)).rejects.toThrow('schema version') + } + await writer.knex('knex_migrations').delete() + await expect(openKnexSnapshotArchiveSource(reader, identity)).rejects.toThrow('schema version') + const next = await reader.openReadSnapshot() + await next.close() +}) + +test('supports a configured migration table/schema and preserves the longest accepted schema name', async () => { + const { reader, writer } = await fixture() + await writer.knex.schema.renameTable('knex_migrations', 'archive_migration_history') + await writer + .knex('archive_migration_history') + .orderBy('id', 'desc') + .limit(1) + .update({ name: 'x'.repeat(256) }) + reader.knex.client.config.migrations = { tableName: 'archive_migration_history', schemaName: 'main' } + const queries: string[] = [] + reader.knex.on('query', query => queries.push(query.sql)) + const source = await openKnexSnapshotArchiveSource(reader, identity) + expect(queries.some(sql => sql.includes('`main`.`archive_migration_history`'))).toBe(true) + expect(source.sourceSchema).toBe('x'.repeat(256)) + expect(source.isOpen).toBe(true) + await source.close() + await source.closed + expect(source.isOpen).toBe(false) + await writer.knex('archive_migration_history').update({ name: 'x' }) + const shortest = await openKnexSnapshotArchiveSource(reader, identity) + expect(shortest.sourceSchema).toBe('x') + await shortest.close() +}) + +test('a full page continues through its empty terminal page and preserves mainnet binding', async () => { + const { reader, writer, userId } = await fixture() + await writer.knex('settings').update({ chain: 'main' }) + await writer.knex('tx_labels_map').whereIn('txLabelId', [1, 3]).delete() + await writer.knex('tx_labels').where({ userId }).delete() + for (let index = 0; index < 256; index++) await writer.findOrInsertTxLabel(userId, `page-${index}`) + const manifest = await captureKnexSnapshotArchive(reader, writer.knex, identity, 'main') + expect(manifest.binding.sourceStorage.chain).toBe('main') + expect(manifest.pages).toBe(15) + const store = new KnexSnapshotArchiveStore(writer.knex) + expect(await store.read(identity, manifest.archiveId, 8)).toMatchObject({ table: 'txLabels', rows: 128, done: false }) + expect(await store.read(identity, manifest.archiveId, 9)).toMatchObject({ table: 'txLabels', rows: 128, done: false }) + expect(await store.read(identity, manifest.archiveId, 10)).toMatchObject({ table: 'txLabels', rows: 0, done: true }) + await store.close(identity, manifest.archiveId) +}) + +test('oversized source rows and exhausted reservations fail closed', async () => { + const { reader, writer } = await fixture() + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test', { maxBytes: 4100 })).rejects.toThrow( + 'metadata' + ) + await writer + .knex('proven_txs') + .where({ provenTxId: 1 }) + .update({ rawTx: Buffer.alloc(200000) }) + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test')).rejects.toThrow('maxBytes') + expect(await writer.knex('snapshot_archives')).toHaveLength(0) + expect(await writer.knex('snapshot_archive_pages')).toHaveLength(0) +}) + +test('encoding refusal cleans the reservation without returning an oversized frame', async () => { + const { reader, writer } = await fixture() + jest.spyOn(Transfer, 'encodeSyncTransfer').mockReturnValue(new Uint8Array(1048577)) + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test')).rejects.toThrow('encoded page limit') + expect(await writer.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('capture bounds page work even if a source never reports completion', async () => { + const { reader, writer } = await fixture() + const source = await openKnexSnapshotArchiveSource(reader, identity) + const read = jest.fn(async () => ({ rows: [], payloadBytes: 0, done: false })) + jest.spyOn(ArchiveSource, 'openKnexSnapshotArchiveSource').mockResolvedValue({ ...source, readPage: read }) + const append = jest.spyOn(KnexSnapshotArchiveStore.prototype, 'append').mockResolvedValue() + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test')).rejects.toThrow('page limit') + expect(read).toHaveBeenCalledTimes(4096) + expect(append).toHaveBeenCalledTimes(4096) + expect(source.isOpen).toBe(false) + expect(await writer.knex('snapshot_archives')).toHaveLength(0) +}) + +test('normalizes cross-realm dates and rejects malformed dates before staging them', async () => { + const { reader, writer } = await fixture() + for (const value of [runInNewContext('new Date("2026-01-01T00:00:00.000Z")'), new Date(NaN)]) { + const source = await openKnexSnapshotArchiveSource(reader, identity) + const readPage = source.readPage + jest.spyOn(ArchiveSource, 'openKnexSnapshotArchiveSource').mockResolvedValue({ + ...source, + readPage: async (...args) => { + const page = await readPage(...args) + for (const row of page.rows) row.created_at = value + return page + } + }) + const pending = captureKnexSnapshotArchive(reader, writer.knex, identity, 'test') + if (Number.isNaN(value.getTime())) await expect(pending).rejects.toThrow('invalid date') + else { + const manifest = await pending + const store = new KnexSnapshotArchiveStore(writer.knex) + const frame = decodeSyncTransfer((await store.read(identity, manifest.archiveId, 0)).bytes) as { + rows: Array<{ created_at: string }> + } + expect(frame.rows[0].created_at).toBe('2026-01-01T00:00:00.000Z') + await store.close(identity, manifest.archiveId) + } + jest.restoreAllMocks() + } +}) + +test('failed source cleanup prevents sealing and interrupted staging cleanup keeps its reservation', async () => { + const { reader, writer } = await fixture() + const source = await openKnexSnapshotArchiveSource(reader, identity) + const failure = new Error('synthetic source close failure') + jest.spyOn(ArchiveSource, 'openKnexSnapshotArchiveSource').mockResolvedValue({ + ...source, + close: async () => { + await source.close() + throw failure + } + }) + await expect(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test')).rejects.toBe(failure) + expect(await writer.knex('snapshot_archives')).toHaveLength(0) + jest.restoreAllMocks() + const callbackFailure = new Error('synthetic callback failure') + jest.spyOn(KnexSnapshotArchiveStore.prototype, 'close').mockRejectedValue(new Error('synthetic cleanup failure')) + await expect( + captureKnexSnapshotArchive(reader, writer.knex, identity, 'test', { + onProgress: () => { + throw callbackFailure + } + }) + ).rejects.toBe(callbackFailure) + expect(await writer.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 1 }) + expect((await writer.knex('snapshot_archives').first()).state).toBe('building') + jest.restoreAllMocks() + await writer.knex('snapshot_archives').update({ expiresAt: 0 }) + await new KnexSnapshotArchiveStore(writer.knex).reap() + expect(await writer.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0 }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveClosure.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveClosure.ts new file mode 100644 index 000000000..cab1129c2 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveClosure.ts @@ -0,0 +1,159 @@ +import type { SnapshotIndexState } from '../../schema/snapshotSqliteIndexState' +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { walletSnapshotSourceQuery } from '../KnexWalletReadSnapshot' +import type { WalletSnapshotTable } from '../WalletReadSnapshot' + +interface Reference { + table: WalletSnapshotTable + source: string + field: string + target: string + key: string + profile: boolean + optional?: boolean +} + +// Use exactly the same selected rows as the page reader. Global proofs are +// included only through this profile's transactions or proof requests. +const references: readonly Reference[] = [ + { + table: 'transactions', + source: 'transactions', + field: 'provenTxId', + target: 'proven_txs', + key: 'provenTxId', + profile: false, + optional: true + }, + { + table: 'provenTxReqs', + source: 'proven_tx_reqs', + field: 'provenTxId', + target: 'proven_txs', + key: 'provenTxId', + profile: false, + optional: true + }, + { + table: 'commissions', + source: 'commissions', + field: 'transactionId', + target: 'transactions', + key: 'transactionId', + profile: true + }, + { + table: 'outputs', + source: 'outputs', + field: 'transactionId', + target: 'transactions', + key: 'transactionId', + profile: true + }, + { + table: 'outputs', + source: 'outputs', + field: 'basketId', + target: 'output_baskets', + key: 'basketId', + profile: true, + optional: true + }, + { + table: 'outputs', + source: 'outputs', + field: 'spentBy', + target: 'transactions', + key: 'transactionId', + profile: true, + optional: true + }, + { + table: 'txLabelMaps', + source: 'tx_labels_map', + field: 'transactionId', + target: 'transactions', + key: 'transactionId', + profile: true + }, + { + table: 'txLabelMaps', + source: 'tx_labels_map', + field: 'txLabelId', + target: 'tx_labels', + key: 'txLabelId', + profile: true + }, + { + table: 'outputTagMaps', + source: 'output_tags_map', + field: 'outputId', + target: 'outputs', + key: 'outputId', + profile: true + }, + { + table: 'outputTagMaps', + source: 'output_tags_map', + field: 'outputTagId', + target: 'output_tags', + key: 'outputTagId', + profile: true + }, + { + table: 'certificateFields', + source: 'certificate_fields', + field: 'certificateId', + target: 'certificates', + key: 'certificateId', + profile: true + }, + { + table: 'certificateFields', + source: 'certificate_fields', + field: 'userId', + target: 'users', + key: 'userId', + profile: true + } +] + +/** + * Check relational closure in the caller's retained read transaction. Only a + * constant marker is returned for an invalid relation; blobs and full ID maps + * are never loaded. This does not parse or authenticate BRC-38/39 documents. + */ +export async function assertKnexSnapshotArchiveClosure( + k: Knex, + userId: number, + profileIndexes: SnapshotIndexState = false, + relationIndexes: SnapshotIndexState = false, + certificateIndexes: SnapshotIndexState = false, + globalIndexes: SnapshotIndexState = false +): Promise { + if (!Number.isSafeInteger(userId) || userId < 1) throw new WERR_INVALID_PARAMETER('userId', 'a positive safe ID') + await runInSeries(references, async reference => { + const column = `${reference.source}.${reference.field}` + const target = k(reference.target) + .select(k.raw('1')) + .whereRaw('?? = ??', [`${reference.target}.${reference.key}`, column]) + if (reference.profile) void target.where(`${reference.target}.userId`, userId) + const invalid = walletSnapshotSourceQuery( + k, + reference.table, + userId, + profileIndexes, + relationIndexes, + certificateIndexes, + globalIndexes + ) + .select(k.raw('1 AS invalid')) + .whereNotExists(target) + if (reference.optional === true) void invalid.whereNotNull(column) + if ((await invalid.first()) !== undefined) { + throw new WERR_INVALID_OPERATION('Snapshot source contains an incomplete or cross-profile relation') + } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.test.ts new file mode 100644 index 000000000..1ddccca50 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.test.ts @@ -0,0 +1,908 @@ +import { + addSnapshotArchiveOwnerTable, + removeSnapshotArchiveOwnerTable +} from '../../schema/snapshotArchiveOwnerMigration' +import { SnapshotArchiveCleanupPendingError } from './SnapshotArchiveOwner' +import { snapshotArchiveReaderRequestId } from './SnapshotArchiveReaderRequest' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { addSnapshotArchiveTables } from '../../schema/snapshotArchiveMigration' +import { + addSnapshotArchiveRequestTable, + removeSnapshotArchiveRequestTable +} from '../../schema/snapshotArchiveRequestMigration' +import { KnexSnapshotArchiveStore } from './KnexSnapshotArchiveStore' +import { KnexSnapshotArchiveRequestStore } from './KnexSnapshotArchiveRequestStore' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import * as ArchiveSql from './SnapshotArchiveSql' +import { snapshotArchiveTables, type SnapshotArchiveBinding, type SnapshotArchiveWriter } from './SnapshotArchive' + +const identity = '02' + '11'.repeat(32) +const other = '03' + '22'.repeat(32) +const date = new Date('2026-01-01T00:00:00.000Z') +const binding: SnapshotArchiveBinding = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceSchema: 'request-fixture-v1', + sourceStorage: { + created_at: date, + updated_at: date, + storageIdentityKey: 'source', + storageName: 'source', + chain: 'test', + dbtype: 'SQLite', + maxOutputScript: 1024 + }, + user: { created_at: date, updated_at: date, userId: 1, identityKey: identity, activeStorage: 'source' } +} +const databases: Knex[] = [] +const directories: string[] = [] + +async function fixture(requireSourceDrain = false) { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-request-')) + directories.push(directory) + const open = () => { + const db = knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'requests.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 5000 + }) + databases.push(db) + return db + } + const db = open() + await db.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(db) + await addSnapshotArchiveRequestTable(db) + if (requireSourceDrain) await addSnapshotArchiveOwnerTable(db) + const peer = open() + return { + db, + peer, + requests: new KnexSnapshotArchiveRequestStore(db, requireSourceDrain), + second: new KnexSnapshotArchiveRequestStore(peer, requireSourceDrain), + archives: new KnexSnapshotArchiveStore(db) + } +} + +function request(nonce = 'b'.repeat(64), maxBytes = 32768) { + const value = { version: 1 as const, nonce, notAfter: Date.now() + 300000, maxBytes } + return { ...value, requestId: snapshotArchiveRequestId(value) } +} + +async function append(archives: KnexSnapshotArchiveStore, writer: SnapshotArchiveWriter) { + for (const [sequence, table] of snapshotArchiveTables.entries()) { + await archives.append(writer, { sequence, table, rows: 0, done: true, bytes: Uint8Array.of(0) }) + } +} + +afterEach(async () => { + jest.restoreAllMocks() + await Promise.all(databases.splice(0).map(db => db.destroy())) + await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))) +}) + +test('lost creation acknowledgements recover the same sealed archive on an independent server', async () => { + const { db, requests, second, archives } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + expect(admitted.receipt).toEqual({ + version: 1, + requestId: input.requestId, + expiresAt: input.notAfter, + state: 'building' + }) + expect(admitted.owner).toBeDefined() + expect(await second.claim(identity, input)).toEqual({ receipt: admitted.receipt }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: input.maxBytes }) + const writer = await requests.begin(admitted.owner!, binding) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: input.maxBytes }) + await append(archives, writer) + const manifest = await requests.seal(admitted.owner!, writer) + expect(manifest.expiresAt).toBe(input.notAfter) + const recovered = await second.claim(identity, input) + expect(recovered).toEqual({ + receipt: { ...admitted.receipt, state: 'ready', archiveId: writer.archiveId, digest: manifest.digest } + }) + expect(await second.status(identity, input.requestId)).toEqual(recovered.receipt) + expect(JSON.stringify(recovered)).not.toContain(admitted.owner!.claimToken) + expect(JSON.stringify(recovered)).not.toContain(writer.writerToken) + await second.close(identity, input.requestId) + await second.close(identity, input.requestId) + expect((await requests.claim(identity, input)).receipt.state).toBe('closed') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + await expect(second.status(other, input.requestId)).rejects.toThrow('unavailable') +}) + +test('a pending claim reserves capacity before any source or archive exists and cancellation releases once', async () => { + const { db, peer, requests, second, archives } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + expect(await db('snapshot_archives')).toHaveLength(0) + await expect(second.claim(identity, request('c'.repeat(64)))).rejects.toThrow('occupied') + // Independent synchronous SQLite pools may report SQLITE_BUSY rather than + // yield to another transaction in this event loop. Drain both requests and + // retry the refused one; neither outcome may double-release its reservation. + await db.raw('PRAGMA busy_timeout = 0') + await peer.raw('PRAGMA busy_timeout = 0') + const outcomes = await Promise.allSettled([ + requests.close(identity, input.requestId), + second.close(identity, input.requestId) + ]) + expect(outcomes.some(result => result.status === 'fulfilled')).toBe(true) + for (const result of outcomes) { + if (result.status === 'rejected') expect(result.reason).toMatchObject({ code: 'SQLITE_BUSY' }) + } + await requests.close(identity, input.requestId) + await second.close(identity, input.requestId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await expect(requests.begin(admitted.owner!, binding)).rejects.toThrow('unavailable') + const local = await archives.begin(binding) + await archives.close(identity, local.archiveId) +}) + +test('failed archive assignment rolls back its capacity handoff and remains safely cancellable', async () => { + const { db, requests } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + await db.raw( + "CREATE TRIGGER fail_request_assignment BEFORE UPDATE ON snapshot_archive_requests WHEN NEW.archiveId IS NOT NULL BEGIN SELECT RAISE(ABORT, 'synthetic assignment interruption'); END" + ) + await expect(requests.begin(admitted.owner!, binding)).rejects.toThrow('synthetic assignment interruption') + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: input.maxBytes }) + expect((await requests.status(identity, input.requestId)).state).toBe('building') + await db.raw('DROP TRIGGER fail_request_assignment') + await requests.close(identity, input.requestId, 'failed') + expect((await requests.status(identity, input.requestId)).state).toBe('failed') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('a failed ready-receipt commit cannot publish the archive separately', async () => { + const { db, requests, archives } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + const writer = await requests.begin(admitted.owner!, binding) + await append(archives, writer) + await db.raw( + "CREATE TRIGGER fail_request_ready BEFORE UPDATE ON snapshot_archive_requests WHEN NEW.state = 'ready' BEGIN SELECT RAISE(ABORT, 'synthetic ready interruption'); END" + ) + await expect(requests.seal(admitted.owner!, writer)).rejects.toThrow('synthetic ready interruption') + expect((await requests.status(identity, input.requestId)).state).toBe('building') + await expect(archives.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') + await db.raw('DROP TRIGGER fail_request_ready') + const manifest = await requests.seal(admitted.owner!, writer) + expect((await requests.status(identity, input.requestId)).digest).toBe(manifest.digest) +}) + +test('pending claims count against ordinary staging capacity and expired claims are reaped', async () => { + const { db, requests, archives } = await fixture() + for (let n = 0; n < 8; n++) + await requests.claim('02' + n.toString(16).padStart(64, '0'), request(n.toString(16).padStart(64, '0'))) + await expect(archives.begin(binding, { maxBytes: 32768 })).rejects.toThrow('occupied') + await expect(requests.claim(identity, request())).rejects.toThrow('occupied') + await db('snapshot_archive_requests').update({ expiresAt: 0 }) + await requests.reap() + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('terminal receipt history remains bounded even after physical capacity is released', async () => { + const { requests, db } = await fixture() + for (let n = 0; n < 4; n++) { + const input = request(n.toString(16).padStart(64, '0')) + await requests.claim(identity, input) + await requests.close(identity, input.requestId) + } + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await expect(requests.claim(identity, request())).rejects.toThrow('occupied') + expect(await db('snapshot_archive_requests')).toHaveLength(4) +}) + +test('schema removal refuses outstanding reservations and repeated creation preserves receipts', async () => { + const { db, requests } = await fixture() + const input = request() + await requests.claim(identity, input) + await addSnapshotArchiveRequestTable(db) + expect((await requests.status(identity, input.requestId)).state).toBe('building') + await expect(removeSnapshotArchiveRequestTable(db)).rejects.toThrow('Close snapshot archive requests') + await requests.close(identity, input.requestId) + await removeSnapshotArchiveRequestTable(db) + await removeSnapshotArchiveRequestTable(db) + expect(await db.schema.hasTable('snapshot_archive_requests')).toBe(false) +}) + +test('an actually expired request stays unavailable after its receipt has been collected', async () => { + const { db, requests } = await fixture() + const input = { ...request(), notAfter: Date.now() + 100 } + input.requestId = snapshotArchiveRequestId(input) + await requests.claim(identity, input) + await new Promise(resolve => setTimeout(resolve, 150)) + expect((await requests.status(identity, input.requestId)).state).toBe('expired') + await requests.reap() + expect(await db('snapshot_archive_requests')).toHaveLength(0) + await expect(requests.claim(identity, input)).rejects.toThrow('Invalid snapshot archive creation request') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('global terminal history bounds many distinct profiles without consuming released capacity', async () => { + const { db, requests } = await fixture() + for (let n = 0; n < 64; n++) { + const owner = '02' + n.toString(16).padStart(64, '0') + const input = request(n.toString(16).padStart(64, '0')) + await requests.claim(owner, input) + await requests.close(owner, input.requestId) + } + await expect(requests.claim(identity, request())).rejects.toThrow('occupied') + expect(await db('snapshot_archive_requests')).toHaveLength(64) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('capture ownership, profile and archive binding must all match before durable effects', async () => { + const { db, requests, archives } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + await expect(requests.begin({ ...admitted.owner!, claimToken: '0'.repeat(64) }, binding)).rejects.toThrow( + 'unavailable' + ) + await expect( + requests.begin(admitted.owner!, { ...binding, user: { ...binding.user, identityKey: other } }) + ).rejects.toThrow('unavailable') + expect(await db('snapshot_archives')).toHaveLength(0) + const writer = await requests.begin(admitted.owner!, binding) + await expect(requests.begin(admitted.owner!, binding)).rejects.toThrow('unavailable') + await append(archives, writer) + await expect(requests.seal(admitted.owner!, { ...writer, archiveId: '0'.repeat(64) })).rejects.toThrow('unavailable') + await expect(requests.seal(admitted.owner!, { ...writer, writerToken: '0'.repeat(64) })).rejects.toThrow( + 'unavailable' + ) + const manifest = await requests.seal(admitted.owner!, writer) + expect(await requests.seal(admitted.owner!, writer)).toEqual(manifest) + await requests.close(identity, input.requestId) + await expect(requests.seal(admitted.owner!, writer)).rejects.toThrow('unavailable') +}) + +test('a local archive that wins a profile race leaves the pending request reservation recoverable', async () => { + const { db, requests, archives } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + const local = await archives.begin(binding, { maxBytes: 8192 }) + await expect(requests.begin(admitted.owner!, binding)).rejects.toThrow('occupied') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 2, reservedBytes: 40960 }) + await requests.close(identity, input.requestId, 'failed') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 8192 }) + await archives.close(identity, local.archiveId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test.each(['', 'x' + identity, identity + 'x', 7, null])( + 'refuses malformed request profile %p before database work', + async value => { + const { requests, db } = await fixture() + await expect(requests.claim(value as string, request())).rejects.toThrow( + /identityKey.*compressed public identity key/ + ) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + } +) + +test.each(['', 'x' + 'a'.repeat(64), 'a'.repeat(64) + 'x', 7, null])( + 'refuses malformed request identifier %p', + async value => { + const { requests } = await fixture() + await expect(requests.status(identity, value as string)).rejects.toThrow( + /requestId.*version-one snapshot archive request identifier/ + ) + } +) + +test('ready profiles retain independent deadlines and cleanup ownership', async () => { + const { db, requests, archives } = await fixture() + const first = request() + const firstClaim = await requests.claim(identity, first) + const firstWriter = await requests.begin(firstClaim.owner!, binding) + await append(archives, firstWriter) + await requests.seal(firstClaim.owner!, firstWriter) + const next = { ...request('c'.repeat(64)), notAfter: first.notAfter - 60000 } + next.requestId = snapshotArchiveRequestId(next) + const nextClaim = await requests.claim(other, next) + const nextWriter = await requests.begin(nextClaim.owner!, { + ...binding, + user: { ...binding.user, identityKey: other } + }) + await append(archives, nextWriter) + const nextManifest = await requests.seal(nextClaim.owner!, nextWriter) + expect((await archives.inspect(identity, firstWriter.archiveId)).expiresAt).toBe(first.notAfter) + expect(nextManifest.expiresAt).toBe(next.notAfter) + await requests.close(identity, first.requestId) + expect((await requests.status(other, next.requestId)).state).toBe('ready') + expect((await db('snapshot_archive_requests').where({ identityKey: other }).first()).released).toBe(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + expect((await archives.inspect(other, nextWriter.archiveId)).digest).toBe(nextManifest.digest) + await requests.close(other, next.requestId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('existing local archives refuse request admission for their profile', async () => { + const { db, requests, archives } = await fixture() + const local = await archives.begin(binding, { maxBytes: 8192 }) + await expect(requests.claim(identity, request())).rejects.toThrow('occupied') + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 8192 }) + await archives.close(identity, local.archiveId) +}) + +test('byte reservations accept the exact global limit and reject the next capture', async () => { + const { db, requests } = await fixture() + for (let n = 0; n < 4; n++) + await requests.claim( + '02' + n.toString(16).padStart(64, '0'), + request(n.toString(16).padStart(64, '0'), 32 * 1024 * 1024) + ) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 4, reservedBytes: 128 * 1024 * 1024 }) + await expect(requests.claim(identity, request())).rejects.toThrow('occupied') + expect(await db('snapshot_archive_requests')).toHaveLength(4) +}) + +test.each(['closed', 'failed', 'expired'] as const)( + 'reaping resumes interrupted %s cleanup without collecting unexpired history', + async state => { + const { db, requests, archives } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + const writer = await requests.begin(admitted.owner!, binding) + await append(archives, writer) + await db.raw( + "CREATE TRIGGER fail_request_cleanup BEFORE DELETE ON snapshot_archive_pages BEGIN SELECT RAISE(ABORT, 'synthetic cleanup interruption'); END" + ) + await expect(requests.close(identity, input.requestId, state)).rejects.toThrow('synthetic cleanup interruption') + expect((await requests.status(identity, input.requestId)).state).toBe(state) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + expect((await db('snapshot_archive_requests').first()).released).toBe(0) + await db.raw('DROP TRIGGER fail_request_cleanup') + await requests.reap() + expect(await db('snapshot_archive_requests')).toHaveLength(1) + expect((await db('snapshot_archive_requests').first()).released).toBe(1) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + } +) + +test('a creation request expires exactly at its immutable database-clock deadline', async () => { + const { db, requests } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + const clock = jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(input.notAfter) + try { + expect((await requests.status(identity, input.requestId)).state).toBe('expired') + await expect(requests.begin(admitted.owner!, binding)).rejects.toThrow('unavailable') + await requests.reap() + expect(await db('snapshot_archive_requests')).toHaveLength(0) + } finally { + clock.mockRestore() + } +}) + +test.each([0, 1])('admission rechecks %i remaining milliseconds after ownership lookup', async remaining => { + const { db, requests } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + const clock = jest + .spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow') + .mockResolvedValueOnce(input.notAfter - 1) + .mockResolvedValue(input.notAfter - remaining) + try { + const pending = requests.begin(admitted.owner!, binding) + if (remaining === 0) { + await expect(pending).rejects.toThrow('unavailable') + expect(await db('snapshot_archives')).toHaveLength(0) + } else { + const writer = await pending + expect((await db('snapshot_archives').where({ archiveId: writer.archiveId }).first()).expiresAt).toBe( + input.notAfter + ) + } + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + } finally { + clock.mockRestore() + } +}) + +test('full-request cancellation fences a delayed first claim on another connection without reserving capacity', async () => { + const { db, requests, second } = await fixture() + const input = request() + await second.markCancellation(identity, input) + const closed = { version: 1, requestId: input.requestId, expiresAt: input.notAfter, state: 'closed' } + expect(await requests.claim(identity, input)).toEqual({ receipt: closed }) + expect(await second.status(identity, input.requestId)).toEqual(closed) + expect(await db('snapshot_archive_requests').first()).toMatchObject({ + state: 'closed', + released: 1, + reservedBytes: 0, + archiveId: null + }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await requests.markCancellation(identity, { ...input }) + await second.close(identity, input.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(1) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + const separate = await requests.claim(other, input) + expect(separate.owner).toBeDefined() + expect(separate.receipt.state).toBe('building') + await requests.close(other, input.requestId) +}) + +test('the cancellation fence retains an existing reservation until ordinary physical cleanup', async () => { + const { db, requests, second } = await fixture() + const input = request() + const admitted = await requests.claim(identity, input) + await second.markCancellation(identity, input) + expect((await requests.status(identity, input.requestId)).state).toBe('closed') + expect(await db('snapshot_archive_requests').first()).toMatchObject({ released: 0, reservedBytes: input.maxBytes }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: input.maxBytes }) + await expect(requests.begin(admitted.owner!, binding)).rejects.toThrow('unavailable') + await second.close(identity, input.requestId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('a cancellation tombstone expires at the original deadline and cannot make that request reusable', async () => { + const { db, requests, second } = await fixture() + const input = request() + await requests.markCancellation(identity, input) + const clock = jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(input.notAfter) + await second.reap() + expect(await db('snapshot_archive_requests')).toHaveLength(0) + await requests.markCancellation(identity, input) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + await expect(second.claim(identity, input)).rejects.toThrow('Invalid snapshot archive creation request') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + clock.mockRestore() +}) + +test('cancellation history exhaustion fails observably without falsely acknowledging a durable fence', async () => { + const { db, requests, second } = await fixture() + const retained = [] + for (let i = 0; i < 4; i++) { + const input = request(i.toString(16).padStart(64, '0')) + retained.push(input) + await requests.markCancellation(identity, input) + } + await expect(second.markCancellation(identity, request())).rejects.toThrow('history is occupied') + expect(await db('snapshot_archive_requests')).toHaveLength(4) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + // A retained request remains idempotently cancellable at the history ceiling. + await second.markCancellation(identity, retained[0]) + expect((await requests.claim(identity, retained[0])).receipt.state).toBe('closed') +}) + +test('failed cancellation persistence rolls back and never creates a partial reservation', async () => { + const { db, requests } = await fixture() + const input = request() + await db.raw( + "CREATE TRIGGER fail_request_cancel BEFORE INSERT ON snapshot_archive_requests WHEN NEW.state = 'closed' BEGIN SELECT RAISE(ABORT, 'synthetic cancellation interruption'); END" + ) + await expect(requests.markCancellation(identity, input)).rejects.toThrow('synthetic cancellation interruption') + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await db.raw('DROP TRIGGER fail_request_cancel') + await requests.markCancellation(identity, input) + expect((await requests.claim(identity, input)).receipt.state).toBe('closed') +}) + +// Append to RequestStore tests after implementing the offer. Uses real existing +// independent SQLite fixture and assertions against independently read tables. +test('server offers retain bounded cancellation ownership without charging a capture', async () => { + const { db, requests, second } = await fixture() + const options = { lifetimeMs: 300000, maxBytes: 32768 } + const issued = (await requests.offer(identity, options))! + expect(issued.request.notAfter).toBe(issued.serverTime + options.lifetimeMs) + expect(issued.request.maxBytes).toBe(options.maxBytes) + expect(issued.request.requestId).toBe(snapshotArchiveReaderRequestId(issued.request)) + expect(await db('snapshot_archive_requests').first()).toMatchObject({ + identityKey: identity, + state: 'offered', + reservedBytes: 0, + released: 1, + archiveId: null + }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await db('snapshot_archives')).toHaveLength(0) + await second.markReaderCancellation(identity, issued.request) + await second.close(identity, issued.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + await expect(requests.claimReader(identity, issued.request)).rejects.toThrow('unavailable') + await expect(requests.claim(identity, issued.request)).rejects.toThrow('Invalid snapshot archive creation request') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('a retained offer can be admitted when every history slot is occupied, charging exactly once', async () => { + const { db, requests, second } = await fixture() + const offers = [] + for (let n = 0; n < 4; n++) offers.push((await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))!) + expect(new Set(offers.map(offer => offer.request.requestId)).size).toBe(4) + expect(await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 })).toBeUndefined() + expect(await db('snapshot_archive_requests')).toHaveLength(4) + const admitted = await second.claimReader(identity, offers[0].request) + expect(admitted.owner).toBeDefined() + expect(await requests.claimReader(identity, offers[0].request)).toEqual({ receipt: admitted.receipt }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + await expect(requests.claimReader(identity, offers[1].request)).rejects.toThrow('occupied') + await requests.markReaderCancellation(identity, offers[1].request) + await requests.close(identity, offers[1].request.requestId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + await second.close(identity, offers[0].request.requestId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('lost offers expire without source or capture quota and cannot be admitted after collection', async () => { + const { db, requests } = await fixture() + const clock = jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(1000000) + const issued = (await requests.offer(identity, { lifetimeMs: 50, maxBytes: 32768 }))! + clock.mockResolvedValue(1000050) + await requests.reap() + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await expect(requests.claimReader(identity, issued.request)).rejects.toThrow( + 'Invalid snapshot archive reader request' + ) +}) + +test('failed offered-to-claimed persistence rolls back quota and remains cancellable', async () => { + const { db, requests, second } = await fixture() + const issued = (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + await db.raw( + "CREATE TRIGGER fail_offer_claim BEFORE UPDATE ON snapshot_archive_requests WHEN NEW.state = 'claimed' BEGIN SELECT RAISE(ABORT, 'synthetic offered admission interruption'); END" + ) + await expect(second.claimReader(identity, issued.request)).rejects.toThrow('synthetic offered admission interruption') + expect(await db('snapshot_archive_requests').first()).toMatchObject({ + state: 'offered', + released: 1, + reservedBytes: 0 + }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await db.raw('DROP TRIGGER fail_offer_claim') + await requests.markReaderCancellation(identity, issued.request) + await requests.close(identity, issued.request.requestId) + await expect(second.claimReader(identity, issued.request)).rejects.toThrow('unavailable') + expect(await db('snapshot_archive_requests')).toHaveLength(0) +}) + +test('frequent completed readers release history immediately without reopening delayed requests', async () => { + const { db, requests, second } = await fixture() + const old = [] + for (let n = 0; n < 16; n++) { + const offer = (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + old.push(offer.request) + const admitted = await requests.claimReader(identity, offer.request) + expect(admitted.owner).toBeDefined() + await second.markReaderCancellation(identity, offer.request) + await second.close(identity, offer.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + } + for (const request of old) { + await expect(requests.claimReader(identity, request)).rejects.toThrow('unavailable') + await expect(requests.claim(identity, request)).rejects.toThrow('Invalid snapshot archive creation request') + await expect(second.markReaderCancellation(identity, request)).resolves.toBeUndefined() + } + expect(await db('snapshot_archive_requests')).toHaveLength(0) +}) + +test('a caller cannot admit a reader tuple that the server never offered', async () => { + const { db, requests } = await fixture() + const fields = { version: 2 as const, nonce: 'a'.repeat(64), notAfter: Date.now() + 300000, maxBytes: 32768 } + const input = { ...fields, requestId: snapshotArchiveReaderRequestId(fields) } + await expect(requests.claimReader(identity, input)).rejects.toThrow('unavailable') + await expect(requests.claim(identity, input)).rejects.toThrow('Invalid snapshot archive creation request') + await expect(requests.markReaderCancellation(identity, input)).resolves.toBeUndefined() + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('reader cleanup keeps its reservation and fence through a failed page deletion', async () => { + const { db, requests, second, archives } = await fixture() + const offered = (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + const admitted = await requests.claimReader(identity, offered.request) + const writer = await requests.begin(admitted.owner!, binding) + await append(archives, writer) + await requests.seal(admitted.owner!, writer) + await db.raw( + "CREATE TRIGGER fail_reader_cleanup BEFORE DELETE ON snapshot_archive_pages BEGIN SELECT RAISE(ABORT, 'synthetic reader cleanup interruption'); END" + ) + await expect(second.close(identity, offered.request.requestId)).rejects.toThrow( + 'synthetic reader cleanup interruption' + ) + expect(await db('snapshot_archive_requests').first()).toMatchObject({ + state: 'closed', + released: 0, + archiveId: writer.archiveId + }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + expect(await requests.claimReader(identity, offered.request)).toEqual({ + receipt: { version: 1, requestId: offered.request.requestId, expiresAt: offered.request.notAfter, state: 'closed' } + }) + expect(await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 })).toBeUndefined() + await db.raw('DROP TRIGGER fail_reader_cleanup') + await second.close(identity, offered.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await expect(requests.claimReader(identity, offered.request)).rejects.toThrow('unavailable') +}) + +test('reader offers remain profile-bound through claim and cancellation', async () => { + const { db, requests, second } = await fixture() + const issued = (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + await expect(second.claimReader(other, issued.request)).rejects.toThrow('unavailable') + await second.markReaderCancellation(other, issued.request) + expect(await db('snapshot_archive_requests')).toHaveLength(1) + expect(await db('snapshot_archive_requests').first()).toMatchObject({ identityKey: identity, state: 'offered' }) + expect((await requests.claimReader(identity, issued.request)).owner).toBeDefined() + await second.close(identity, issued.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) +}) + +test('early reader collection refuses altered persisted tuple binding and rolls back release', async () => { + const { db, requests } = await fixture() + const issued = (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + await requests.claimReader(identity, issued.request) + await db('snapshot_archive_requests').update({ + requestJson: JSON.stringify({ ...issued.request, requestId: '0'.repeat(64) }) + }) + await expect(requests.close(identity, issued.request.requestId)).rejects.toThrow( + 'Invalid snapshot archive reader request' + ) + expect(await db('snapshot_archive_requests').first()).toMatchObject({ state: 'closed', released: 0 }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + await db('snapshot_archive_requests').update({ requestJson: JSON.stringify(issued.request) }) + await requests.close(identity, issued.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test.each(['cancel', 'expiry'] as const)( + '%s cannot release another source owner before its physical acknowledgement', + async reason => { + const { db, requests, second } = await fixture(true) + const input = request() + const { owner } = await requests.claim(identity, input) + expect(await db('snapshot_archive_owners')).toEqual([{ slot: 0, ...owner, archiveId: null }]) + await expect(removeSnapshotArchiveOwnerTable(db)).rejects.toThrow('Drain snapshot archive sources') + if (reason === 'expiry') jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(input.notAfter) + await expect( + second.close(identity, input.requestId, reason === 'expiry' ? 'expired' : 'closed') + ).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + await second.reap() + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + await requests.sourceClosed({ ...owner!, claimToken: 'f'.repeat(64) }) + expect(await db('snapshot_archive_owners')).toHaveLength(1) + await expect(second.close(identity, input.requestId)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + // The exact owner's acknowledgement remains valid after expiry. + await requests.sourceClosed(owner!) + await requests.sourceClosed(owner!) + await second.close(identity, input.requestId) + expect(await db('snapshot_archive_owners')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await removeSnapshotArchiveOwnerTable(db) + expect(await db.schema.hasTable('snapshot_archive_owners')).toBe(false) + } +) + +test('request cancellation fences the next atomic append and direct archive reaping cannot bypass source ownership', async () => { + const { db, peer, requests, second, archives } = await fixture(true) + const input = request() + const { owner } = await requests.claim(identity, input) + const writer = await requests.begin(owner!, binding) + const page = { sequence: 0, table: snapshotArchiveTables[0], rows: 0, done: true, bytes: Uint8Array.of(7) } + const mutableWriter = { ...writer } + const appended = requests.append(owner!, mutableWriter, page) + mutableWriter.archiveId = 'f'.repeat(64) + page.bytes[0] = 9 + await appended + expect(new Uint8Array((await db('snapshot_archive_pages').first()).payload)).toEqual(Uint8Array.of(7)) + expect(await db('snapshot_archive_owners').first()).toMatchObject({ ...owner, archiveId: writer.archiveId }) + await second.markCancellation(identity, input) + await expect( + requests.append(owner!, writer, { ...page, sequence: 1, table: snapshotArchiveTables[1] }) + ).rejects.toThrow('unavailable') + await expect(new KnexSnapshotArchiveStore(peer).close(identity, writer.archiveId)).rejects.toBeInstanceOf( + SnapshotArchiveCleanupPendingError + ) + await second.reap() + await archives.reap() + expect(await db('snapshot_archive_pages')).toHaveLength(1) + expect(await db('snapshot_archives').first()).toMatchObject({ state: 'closing' }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + await requests.sourceClosed(owner!) + await second.close(identity, input.requestId) + await second.close(identity, input.requestId) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('a guarded request cannot publish a ready receipt until its source is physically acknowledged closed', async () => { + const { db, requests, second, archives } = await fixture(true) + const input = request() + const { owner } = await requests.claim(identity, input) + const writer = await requests.begin(owner!, binding) + for (const [sequence, table] of snapshotArchiveTables.entries()) + await requests.append(owner!, writer, { sequence, table, rows: 0, done: true, bytes: Uint8Array.of(1) }) + await expect(requests.seal(owner!, writer)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect((await second.status(identity, input.requestId)).state).toBe('building') + await expect(archives.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') + await requests.sourceClosed(owner!) + const manifest = await requests.seal(owner!, writer) + expect(await second.status(identity, input.requestId)).toMatchObject({ + state: 'ready', + archiveId: writer.archiveId, + digest: manifest.digest + }) + expect(await db('snapshot_archive_owners')).toHaveLength(0) + await second.close(identity, input.requestId) +}) + +test('an occupied source-owner table refuses admission even if its capacity ledger was independently damaged', async () => { + const { db, requests } = await fixture(true) + await db('snapshot_archive_owners').insert( + Array.from({ length: 8 }, (_, slot) => ({ + slot, + identityKey: other, + requestId: slot.toString(16).padStart(64, '0'), + claimToken: 'e'.repeat(64), + archiveId: null + })) + ) + await expect(requests.claim(identity, request())).rejects.toThrow('source capacity is occupied') + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await db('snapshot_archive_owners')).toHaveLength(8) +}) + +test('source slots are reused only after acknowledgement and stale owners cannot release a successor', async () => { + const { db, requests } = await fixture(true) + const first = request() + const firstClaim = await requests.claim(identity, first) + await requests.sourceClosed(firstClaim.owner!) + await requests.close(identity, first.requestId) + const next = request('c'.repeat(64)) + const nextClaim = await requests.claim(identity, next) + expect(await db('snapshot_archive_owners').first()).toMatchObject({ slot: 0, ...nextClaim.owner }) + await requests.sourceClosed(firstClaim.owner!) + await expect(requests.close(identity, next.requestId)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect((await db('snapshot_archive_capacity').first()).archives).toBe(1) + await requests.sourceClosed(nextClaim.owner!) + await requests.close(identity, next.requestId) + expect((await db('snapshot_archive_capacity').first()).archives).toBe(0) +}) + +test('source-owner schema creation and removal are idempotent without discarding occupied slots', async () => { + const { db, requests } = await fixture(true) + const input = request() + const { owner } = await requests.claim(identity, input) + const rows = await db('snapshot_archive_owners') + await addSnapshotArchiveOwnerTable(db) + expect(await db('snapshot_archive_owners')).toEqual(rows) + await requests.sourceClosed(owner!) + await requests.close(identity, input.requestId) + await removeSnapshotArchiveOwnerTable(db) + await removeSnapshotArchiveOwnerTable(db) + expect(await db.schema.hasTable('snapshot_archive_owners')).toBe(false) +}) + +test('a ready request with no bound archive refuses its receipt without releasing capacity', async () => { + const { db, requests } = await fixture() + const input = request() + await requests.claim(identity, input) + await db('snapshot_archive_requests').update({ state: 'ready' }) + await expect(requests.status(identity, input.requestId)).rejects.toThrow('Snapshot archive request is unavailable') + expect((await db('snapshot_archive_capacity').first()).archives).toBe(1) + expect((await db('snapshot_archive_requests').first()).released).toBe(0) +}) + +test.each(['claim', 'cancel', 'reader-cancel'] as const)( + '%s rejects an altered persisted immutable request tuple without changing its reservation', + async operation => { + const { db, requests } = await fixture() + const input = + operation === 'reader-cancel' + ? (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))!.request + : request() + if (operation === 'reader-cancel') await requests.claimReader(identity, input) + else await requests.claim(identity, input) + await db('snapshot_archive_requests').update({ requestJson: JSON.stringify({ ...input, maxBytes: 32769 }) }) + const action = + operation === 'claim' + ? requests.claim(identity, input) + : operation === 'cancel' + ? requests.markCancellation(identity, input) + : requests.markReaderCancellation(identity, input) + await expect(action).rejects.toThrow('Snapshot archive request is unavailable') + expect((await db('snapshot_archive_requests').first()).state).toBe('claimed') + expect((await db('snapshot_archive_capacity').first()).archives).toBe(1) + } +) + +test.each(['request-id', 'deadline'] as const)( + 'reader collection compares a valid decoded tuple against its persisted %s and rolls back release', + async field => { + const { db, requests } = await fixture() + const issued = (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + await requests.claimReader(identity, issued.request) + const fields = { ...issued.request, nonce: 'c'.repeat(64) } + const different = { ...fields, requestId: snapshotArchiveReaderRequestId(fields) } + await db('snapshot_archive_requests').update( + field === 'request-id' ? { requestJson: JSON.stringify(different) } : { expiresAt: issued.request.notAfter + 1 } + ) + await expect(requests.close(identity, issued.request.requestId)).rejects.toThrow( + 'Snapshot archive request is unavailable' + ) + expect(await db('snapshot_archive_requests').first()).toMatchObject({ state: 'closed', released: 0 }) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + await db('snapshot_archive_requests').update({ + requestJson: JSON.stringify(issued.request), + expiresAt: issued.request.notAfter + }) + await requests.close(identity, issued.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect((await db('snapshot_archive_capacity').first()).archives).toBe(0) + } +) + +test.each(['null', '0', '[]', '{}'])( + 'legacy terminal metadata %s is retained rather than collected as a version-two reader', + async requestJson => { + const { db, requests } = await fixture() + const input = request() + await requests.claim(identity, input) + await db('snapshot_archive_requests').update({ requestJson }) + await requests.close(identity, input.requestId) + expect(await db('snapshot_archive_requests').first()).toMatchObject({ state: 'closed', released: 1, requestJson }) + expect((await db('snapshot_archive_capacity').first()).archives).toBe(0) + } +) + +test('independent reader closers tolerate receipt collection between archive and request cleanup', async () => { + const { db, requests, second, archives } = await fixture() + const issued = (await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + const { owner } = await requests.claimReader(identity, issued.request) + const writer = await requests.begin(owner!, binding) + await append(archives, writer) + await requests.seal(owner!, writer) + const close = KnexSnapshotArchiveStore.prototype.close + jest.spyOn(KnexSnapshotArchiveStore.prototype, 'close').mockImplementationOnce(async function ( + this: KnexSnapshotArchiveStore, + ...args + ) { + await close.apply(this, args) + await second.close(identity, issued.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + }) + await requests.close(identity, issued.request.requestId) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('reaping exposes an independent database failure and permits a later successful cleanup', async () => { + const { db, requests } = await fixture() + const input = request() + await requests.claim(identity, input) + await requests.markCancellation(identity, input) + await db.raw( + "CREATE TRIGGER fail_reap_release BEFORE UPDATE OF released ON snapshot_archive_requests WHEN NEW.released = 1 BEGIN SELECT RAISE(ABORT, 'synthetic reaper failure'); END" + ) + await expect(requests.reap()).rejects.toThrow('synthetic reaper failure') + expect((await db('snapshot_archive_requests').first()).released).toBe(0) + expect((await db('snapshot_archive_capacity').first()).archives).toBe(1) + await db.raw('DROP TRIGGER fail_reap_release') + await requests.reap() + expect((await db('snapshot_archive_requests').first()).released).toBe(1) + expect((await db('snapshot_archive_capacity').first()).archives).toBe(0) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.ts new file mode 100644 index 000000000..67fe8f721 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.ts @@ -0,0 +1,432 @@ +import { + reserveSnapshotArchiveOwner, + assignSnapshotArchiveOwner, + releaseSnapshotArchiveOwner, + hasSnapshotArchiveOwner, + SnapshotArchiveCleanupPendingError +} from './SnapshotArchiveOwner' +import { Random, Utils } from '@bsv/sdk' +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { SnapshotArchiveAdmissionLimitError } from './SnapshotArchiveAdmission' +import { validateSnapshotArchiveReaderOptions, type SnapshotArchiveReaderOptions } from './SnapshotArchiveReaderOffer' +import { + parseSnapshotArchiveReaderRequest, + validateSnapshotArchiveReaderRequest, + snapshotArchiveReaderRequestId, + type SnapshotArchiveReaderRequest +} from './SnapshotArchiveReaderRequest' +import { KnexSnapshotArchiveStore } from './KnexSnapshotArchiveStore' +import { + snapshotArchiveLimits, + type SnapshotArchiveBinding, + type SnapshotArchiveWriter, + type SnapshotArchiveManifest, + type SnapshotArchivePage +} from './SnapshotArchive' +import { lockSnapshotArchiveCapacity, snapshotArchiveDatabaseNow } from './SnapshotArchiveSql' +import { + parseSnapshotArchiveRequest, + validateSnapshotArchiveRequest, + type SnapshotArchiveRequest, + type SnapshotArchiveRequestOwner, + type SnapshotArchiveRequestReceipt, + type SnapshotArchiveTerminalState +} from './SnapshotArchiveRequest' + +interface RequestRow { + identityKey: string + requestId: string + claimToken: string + state: 'offered' | 'claimed' | 'capturing' | 'ready' | SnapshotArchiveTerminalState + requestJson: string + expiresAt: number | string + reservedBytes: number | string + archiveId: string | null + released: boolean | number +} + +const table = 'snapshot_archive_requests' +export const snapshotArchiveRequestLimits = Object.freeze({ perProfile: 4, total: 64 }) + +function unavailable(): never { + throw new WERR_INVALID_OPERATION('Snapshot archive request is unavailable') +} + +function identity(value: string): void { + if (typeof value !== 'string' || !/^(02|03)[0-9a-fA-F]{64}$/.test(value)) { + throw new WERR_INVALID_PARAMETER('identityKey', 'a compressed public identity key') + } +} + +function identifier(value: string): void { + if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value)) { + throw new WERR_INVALID_PARAMETER('requestId', 'a version-one snapshot archive request identifier') + } +} + +/** Durable deduplication with admission charged before opening a source pool. */ +export class KnexSnapshotArchiveRequestStore { + constructor( + private readonly knex: Knex, + private readonly requireSourceDrain = false, + private readonly guardedSources = false + ) {} + + private async receipt(k: Knex, row: RequestRow): Promise { + const base = { version: 1 as const, requestId: row.requestId, expiresAt: Number(row.expiresAt) } + if (base.expiresAt <= (await snapshotArchiveDatabaseNow(k))) return { ...base, state: 'expired' } + if (row.state === 'ready') { + if (row.archiveId === null) unavailable() + const archive = await new KnexSnapshotArchiveStore(k).inspect(row.identityKey, row.archiveId) + return { ...base, state: 'ready', archiveId: archive.archiveId, digest: archive.digest } + } + if (row.state === 'offered' || row.state === 'claimed' || row.state === 'capturing') + return { ...base, state: 'building' } + return { ...base, state: row.state } + } + + async offer( + identityKey: string, + input: SnapshotArchiveReaderOptions + ): Promise<{ serverTime: number; request: Readonly } | undefined> { + identity(identityKey) + const options = validateSnapshotArchiveReaderOptions(input) + return await this.knex.transaction(async trx => { + const capacity = await lockSnapshotArchiveCapacity(trx) + const now = await snapshotArchiveDatabaseNow(trx) + await trx(table).where('expiresAt', '<=', now).where({ released: true }).delete() + const retained: Array> = await trx(table) + .select('identityKey', 'released') + .limit(snapshotArchiveRequestLimits.total) + const owned = retained.filter(row => row.identityKey === identityKey) + const archive = await trx('snapshot_archives').where({ identityKey }).first('archiveId') + if ( + retained.length >= snapshotArchiveRequestLimits.total || + owned.length >= snapshotArchiveRequestLimits.perProfile || + owned.some(row => !row.released) || + archive !== undefined || + capacity.archives >= snapshotArchiveLimits.archives || + Number(capacity.reservedBytes) + options.maxBytes > snapshotArchiveLimits.totalBytes + ) + return undefined + const fields = { + version: 2 as const, + nonce: Utils.toHex(Random(32)), + notAfter: now + options.lifetimeMs, + maxBytes: options.maxBytes + } + const request = validateSnapshotArchiveReaderRequest( + { ...fields, requestId: snapshotArchiveReaderRequestId(fields) }, + now + ) + const row: RequestRow = { + identityKey, + requestId: request.requestId, + claimToken: '', + state: 'offered', + requestJson: JSON.stringify(request), + expiresAt: request.notAfter, + reservedBytes: 0, + archiveId: null, + released: true + } + await trx(table).insert(row) + return { serverTime: now, request } + }) + } + + async claim( + identityKey: string, + input: unknown + ): Promise<{ receipt: SnapshotArchiveRequestReceipt; owner?: SnapshotArchiveRequestOwner }> { + identity(identityKey) + return await this.claimRequest(identityKey, parseSnapshotArchiveRequest(input), false) + } + + /** Admission never creates an absent reader offer, including after completed cleanup. */ + async claimReader( + identityKey: string, + input: unknown + ): Promise<{ receipt: SnapshotArchiveRequestReceipt; owner?: SnapshotArchiveRequestOwner }> { + identity(identityKey) + return await this.claimRequest(identityKey, parseSnapshotArchiveReaderRequest(input), true) + } + + private async claimRequest( + identityKey: string, + request: Readonly, + reader: boolean + ): Promise<{ receipt: SnapshotArchiveRequestReceipt; owner?: SnapshotArchiveRequestOwner }> { + const requestJson = JSON.stringify(request) + return await this.knex.transaction(async trx => { + const capacity = await lockSnapshotArchiveCapacity(trx) + const now = await snapshotArchiveDatabaseNow(trx) + if (reader) validateSnapshotArchiveReaderRequest(request, now) + else validateSnapshotArchiveRequest(request, now) + const existing: RequestRow | undefined = await trx(table) + .where({ identityKey, requestId: request.requestId }) + .first() + if (existing !== undefined) { + if (existing.requestJson !== requestJson) unavailable() + if (!reader || existing.state !== 'offered') return { receipt: await this.receipt(trx, existing) } + } else if (reader) { + unavailable() + } + await trx(table).where('expiresAt', '<=', now).where({ released: true }).delete() + const retained: Array> = await trx(table) + .select('identityKey', 'released') + .limit(snapshotArchiveRequestLimits.total) + const owned = retained.filter(row => row.identityKey === identityKey) + const archive = await trx('snapshot_archives').where({ identityKey }).first('archiveId') + if ( + (!reader && + (retained.length >= snapshotArchiveRequestLimits.total || + owned.length >= snapshotArchiveRequestLimits.perProfile)) || + owned.some(row => !row.released) || + archive !== undefined || + capacity.archives >= snapshotArchiveLimits.archives || + Number(capacity.reservedBytes) + request.maxBytes > snapshotArchiveLimits.totalBytes + ) + throw new SnapshotArchiveAdmissionLimitError('Snapshot archive request capacity is occupied') + const owner = { identityKey, requestId: request.requestId, claimToken: Utils.toHex(Random(32)) } + const row: RequestRow = { + ...owner, + state: 'claimed', + requestJson, + expiresAt: request.notAfter, + reservedBytes: request.maxBytes, + archiveId: null, + released: false + } + if (reader) await trx(table).where({ identityKey, requestId: request.requestId }).update(row) + else await trx(table).insert(row) + if (this.requireSourceDrain) await reserveSnapshotArchiveOwner(trx, owner, this.guardedSources) + await trx('snapshot_archive_capacity') + .where({ id: 1 }) + .update({ archives: capacity.archives + 1, reservedBytes: Number(capacity.reservedBytes) + request.maxBytes }) + return { receipt: await this.receipt(trx, row), owner } + }) + } + + async status(identityKey: string, requestId: string): Promise { + identity(identityKey) + identifier(requestId) + const row: RequestRow | undefined = await this.knex(table).where({ identityKey, requestId }).first() + if (row === undefined) unavailable() + return await this.receipt(this.knex, row) + } + + private async owned(k: Knex, owner: SnapshotArchiveRequestOwner): Promise { + const row: RequestRow | undefined = await k(table).where(owner).first() + if (row === undefined || row.released || Number(row.expiresAt) <= (await snapshotArchiveDatabaseNow(k))) + unavailable() + return row + } + + /** The claim-to-archive assignment and accounting transition commit together. */ + async begin(owner: SnapshotArchiveRequestOwner, binding: SnapshotArchiveBinding): Promise { + const claim = { ...owner } + return await this.knex.transaction(async trx => { + const capacity = await lockSnapshotArchiveCapacity(trx) + const row = await this.owned(trx, claim) + if (row.state !== 'claimed' || row.archiveId !== null || binding.user.identityKey !== row.identityKey) + unavailable() + const remaining = Number(row.expiresAt) - (await snapshotArchiveDatabaseNow(trx)) + if (remaining < 1) unavailable() + await trx('snapshot_archive_capacity') + .where({ id: 1 }) + .update({ + archives: capacity.archives - 1, + reservedBytes: Number(capacity.reservedBytes) - Number(row.reservedBytes) + }) + const writer = await new KnexSnapshotArchiveStore(trx).begin(binding, { + maxBytes: Number(row.reservedBytes), + lifetimeMs: remaining + }) + await trx('snapshot_archives').where({ archiveId: writer.archiveId }).update({ expiresAt: row.expiresAt }) + await assignSnapshotArchiveOwner(trx, claim, writer.archiveId) + await trx(table).where(claim).update({ state: 'capturing', archiveId: writer.archiveId }) + return writer + }) + } + + /** A remote cancellation fences the next append in its atomic archive transaction. */ + async append( + owner: SnapshotArchiveRequestOwner, + writer: SnapshotArchiveWriter, + page: Omit + ): Promise { + const claim = { ...owner } + const captured = { ...writer } + await new KnexSnapshotArchiveStore(this.knex).append(captured, page, async trx => { + const row = await this.owned(trx, claim) + if (row.state !== 'capturing' || row.archiveId !== captured.archiveId) unavailable() + }) + } + + /** Called only after the local source and its owned pool have physically closed. */ + async sourceClosed(owner: SnapshotArchiveRequestOwner): Promise { + const claim = { ...owner } + await this.knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + await releaseSnapshotArchiveOwner(trx, claim) + }) + } + + async seal(owner: SnapshotArchiveRequestOwner, writer: SnapshotArchiveWriter): Promise { + const claim = { ...owner } + const captured = { ...writer } + return await this.knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + const row = await this.owned(trx, claim) + if (!['capturing', 'ready'].includes(row.state) || row.archiveId !== captured.archiveId) unavailable() + if (await hasSnapshotArchiveOwner(trx, { identityKey: row.identityKey, requestId: row.requestId })) + throw new SnapshotArchiveCleanupPendingError() + const manifest = await new KnexSnapshotArchiveStore(trx).seal(captured) + await trx(table).where(claim).update({ state: 'ready' }) + return manifest + }) + } + + /** Persist a bounded terminal fence before a delayed first admission can reserve resources. */ + async markCancellation(identityKey: string, input: unknown): Promise { + identity(identityKey) + const request = parseSnapshotArchiveRequest(input) + const requestJson = JSON.stringify(request) + await this.knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + const key = { identityKey, requestId: request.requestId } + const existing: RequestRow | undefined = await trx(table).where(key).first() + if (existing !== undefined) { + if (existing.requestJson !== requestJson) unavailable() + if (['claimed', 'capturing', 'ready'].includes(existing.state)) + await trx(table).where(key).update({ state: 'closed' }) + return + } + const now = await snapshotArchiveDatabaseNow(trx) + // The immutable deadline already prevents a future first admission. + if (request.notAfter <= now) return + validateSnapshotArchiveRequest(request, now) + await trx(table).where('expiresAt', '<=', now).where({ released: true }).delete() + const retained: Array> = await trx(table) + .select('identityKey') + .limit(snapshotArchiveRequestLimits.total) + if ( + retained.length >= snapshotArchiveRequestLimits.total || + retained.filter(row => row.identityKey === identityKey).length >= snapshotArchiveRequestLimits.perProfile + ) + throw new SnapshotResourceLimitError('Snapshot archive cancellation history is occupied') + await trx(table).insert({ + ...key, + claimToken: Utils.toHex(Random(32)), + state: 'closed', + requestJson, + expiresAt: request.notAfter, + reservedBytes: 0, + archiveId: null, + released: true + }) + }) + } + + /** Fence only the retained exact reader tuple; absence cannot start another capture. */ + async markReaderCancellation(identityKey: string, input: unknown): Promise { + identity(identityKey) + const request = parseSnapshotArchiveReaderRequest(input) + await this.knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + const key = { identityKey, requestId: request.requestId } + const row: RequestRow | undefined = await trx(table).where(key).first() + if (row === undefined) return + if (row.requestJson !== JSON.stringify(request)) unavailable() + // Explicit client cancellation acknowledges a terminal failure receipt too. + // Until then, polling must still observe failed/resource-limited capture. + await trx(table).where(key).update({ state: 'closed' }) + }) + } + + /** Retain failure receipts for polling; only closed/expired reader tuples may be collected early. */ + private async collectReader(k: Knex, row: RequestRow): Promise { + if (!row.released || !['closed', 'expired'].includes(row.state)) return + const decoded: unknown = JSON.parse(row.requestJson) + if ( + decoded === null || + typeof decoded !== 'object' || + Object.getOwnPropertyDescriptor(decoded, 'version')?.value !== 2 + ) + return + const request = parseSnapshotArchiveReaderRequest(decoded) + if (request.requestId !== row.requestId || request.notAfter !== Number(row.expiresAt)) unavailable() + await k(table).where({ identityKey: row.identityKey, requestId: row.requestId, released: true }).delete() + } + + async close(identityKey: string, requestId: string, state: SnapshotArchiveTerminalState = 'closed'): Promise { + identity(identityKey) + identifier(requestId) + await this.knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + const row: RequestRow | undefined = await trx(table).where({ identityKey, requestId }).first() + if (row !== undefined && ['offered', 'claimed', 'capturing', 'ready'].includes(row.state)) { + await trx(table).where({ identityKey, requestId }).update({ state }) + } + }) + await this.release(identityKey, requestId) + } + + private async release(identityKey: string, requestId: string): Promise { + const archiveId = await this.knex.transaction(async trx => { + const capacity = await lockSnapshotArchiveCapacity(trx) + const row: RequestRow | undefined = await trx(table).where({ identityKey, requestId }).first() + if (row === undefined || !['closed', 'failed', 'expired', 'resource-limited'].includes(row.state)) + return undefined + if (row.released) { + await this.collectReader(trx, row) + return undefined + } + if (await hasSnapshotArchiveOwner(trx, { identityKey, requestId })) throw new SnapshotArchiveCleanupPendingError() + if (row.archiveId !== null) return row.archiveId + await trx('snapshot_archive_capacity') + .where({ id: 1 }) + .update({ + archives: capacity.archives - 1, + reservedBytes: Number(capacity.reservedBytes) - Number(row.reservedBytes) + }) + await trx(table).where({ identityKey, requestId }).update({ released: true }) + await this.collectReader(trx, { ...row, released: true }) + return undefined + }) + if (archiveId === undefined) return + await new KnexSnapshotArchiveStore(this.knex).close(identityKey, archiveId) + await this.knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + const row: RequestRow | undefined = await trx(table).where({ identityKey, requestId, archiveId }).first() + if (row === undefined) return + await trx(table).where({ identityKey, requestId, archiveId }).update({ released: true }) + await this.collectReader(trx, { ...row, released: true }) + }) + } + + async reap(): Promise { + const now = await snapshotArchiveDatabaseNow(this.knex) + const rows: RequestRow[] = await this.knex(table) + .where(query => { + void query.where('expiresAt', '<=', now).orWhereIn('state', ['closed', 'failed', 'expired', 'resource-limited']) + }) + .limit(snapshotArchiveRequestLimits.total) + await runInSeries(rows, async row => { + try { + if (Number(row.expiresAt) <= now) await this.close(row.identityKey, row.requestId, 'expired') + else await this.release(row.identityKey, row.requestId) + } catch (error) { + if (!(error instanceof SnapshotArchiveCleanupPendingError)) throw error + } + }) + await this.knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + await trx(table).where('expiresAt', '<=', now).where({ released: true }).delete() + }) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.test.ts new file mode 100644 index 000000000..ad314f161 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.test.ts @@ -0,0 +1,148 @@ +import { KnexSnapshotArchiveRpc } from './KnexSnapshotArchiveRpc' +import { snapshotArchiveCapabilities } from './SnapshotArchiveProtocol' +import { StorageKnex } from '../../StorageKnex' +import { gate, snapshotHttpFixture } from '../../../../test/utils/snapshotArchiveHttpFixtures' +import type { SnapshotArchiveReaderOffer } from './SnapshotArchiveReaderOffer' +import type { KnexSnapshotArchiveService } from './KnexSnapshotArchiveService' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' + +afterEach(() => jest.restoreAllMocks()) + +test('reader admission capacity refusal returns the exact resource-limited offer without opening a capture', async () => { + const fixture = await snapshotHttpFixture() + const rpc = new KnexSnapshotArchiveRpc(fixture.storage) + try { + const service = Reflect.get(rpc, 'service') as KnexSnapshotArchiveService + const issue = jest.spyOn(service, 'offerReader').mockResolvedValue(undefined) + const open = jest.spyOn(fixture.storage, 'openSnapshotArchiveSource') + const options = { lifetimeMs: 300000, maxBytes: 32768 } + expect( + await rpc.dispatch( + 'getSnapshotArchiveReaderOffer', + [{ version: 1, identityKey: fixture.identityKey, options }], + fixture.identityKey + ) + ).toEqual({ version: 1, outcome: 'resource-limited' }) + expect(issue).toHaveBeenCalledTimes(1) + expect(issue).toHaveBeenCalledWith(fixture.identityKey, options) + expect(open).not.toHaveBeenCalled() + } finally { + await rpc.close() + await fixture.close() + } +}) + +test.each([Error, WERR_INVALID_OPERATION])( + 'a %p cancellation failure cannot impersonate a pending receipt', + async ErrorType => { + const fixture = await snapshotHttpFixture() + const rpc = new KnexSnapshotArchiveRpc(fixture.storage) + try { + const issued = (await rpc.dispatch( + 'getSnapshotArchiveReaderOffer', + [{ version: 1, identityKey: fixture.identityKey, options: { lifetimeMs: 300000, maxBytes: 32768 } }], + fixture.identityKey + )) as SnapshotArchiveReaderOffer + if (issued.outcome !== 'offered') throw new Error('Fixture reader was not offered') + const failure = new ErrorType('Snapshot archive source cleanup is pending') + const service = Reflect.get(rpc, 'service') as KnexSnapshotArchiveService + jest.spyOn(service, 'cancelReader').mockRejectedValue(failure) + await expect( + rpc.dispatch( + 'cancelSnapshotArchiveRequest', + [{ version: 1, identityKey: fixture.identityKey, request: issued.request }], + fixture.identityKey + ) + ).rejects.toBe(failure) + } finally { + await rpc.close() + await fixture.close() + } + } +) + +test.each([ + 'snapshot_archive_requests', + 'snapshot_archive_owners', + 'snapshot_archives', + 'snapshot_archive_pages', + 'snapshot_archive_capacity' +])('declines capability when migration table %s is missing', async table => { + const fixture = await snapshotHttpFixture() + const rpc = new KnexSnapshotArchiveRpc(fixture.storage) + try { + expect(await rpc.capabilities()).toEqual(snapshotArchiveCapabilities) + await fixture.storage.knex.schema.dropTable(table) + expect(await rpc.capabilities()).toBeUndefined() + await expect( + rpc.dispatch('getSnapshotArchiveOffer', [{ version: 1, identityKey: fixture.identityKey }], fixture.identityKey) + ).rejects.toThrow('unavailable') + } finally { + await rpc.close() + await fixture.close() + } +}) + +test('unsupported source and a close during capability probing never advertise or open a pool', async () => { + const fixture = await snapshotHttpFixture() + const rpc = new KnexSnapshotArchiveRpc(fixture.storage) + try { + const open = jest.spyOn(fixture.storage, 'openSnapshotArchiveSource') + const supported = jest.spyOn(fixture.storage, 'supportsSnapshotArchiveSource').mockResolvedValue(false) + expect(await rpc.capabilities()).toBeUndefined() + const entered = gate() + const resume = gate() + supported.mockImplementation(async () => { + entered.resolve() + await resume.promise + return true + }) + const pending = rpc.capabilities() + await entered.promise + await rpc.close() + resume.resolve() + expect(await pending).toBeUndefined() + expect(await rpc.capabilities()).toBeUndefined() + expect(open).not.toHaveBeenCalled() + } finally { + await rpc.close() + await fixture.close() + } +}) + +test('unsupported chain is declined and remains checked before an offer', async () => { + const storage = { + chain: 'mock', + knex: {}, + awaitSnapshotArchiveRecovery: async () => undefined + } as unknown as StorageKnex + const rpc = new KnexSnapshotArchiveRpc(storage) + expect(await rpc.capabilities()).toBeUndefined() + jest.spyOn(rpc, 'capabilities').mockResolvedValue(snapshotArchiveCapabilities) + const identityKey = '02' + '11'.repeat(32) + await expect(rpc.dispatch('getSnapshotArchiveOffer', [{ version: 1, identityKey }], identityKey)).rejects.toThrow( + 'chain is unavailable' + ) + await rpc.close() +}) + +test('main-chain sources negotiate and return a main-chain offer without opening a capture', async () => { + const fixture = await snapshotHttpFixture() + Reflect.set(fixture.storage, 'chain', 'main') + const rpc = new KnexSnapshotArchiveRpc(fixture.storage) + try { + const open = jest.spyOn(fixture.storage, 'openSnapshotArchiveSource') + expect(await rpc.capabilities()).toEqual(snapshotArchiveCapabilities) + expect( + await rpc.dispatch( + 'getSnapshotArchiveOffer', + [{ version: 1, identityKey: fixture.identityKey }], + fixture.identityKey + ) + ).toMatchObject({ chain: 'main', sourceStorageIdentityKey: 'http-snapshot-source' }) + expect(open).not.toHaveBeenCalled() + } finally { + await rpc.close() + await fixture.close() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.ts new file mode 100644 index 000000000..019127b52 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.ts @@ -0,0 +1,115 @@ +import type { SnapshotArchiveReaderOffer } from './SnapshotArchiveReaderOffer' +import { SnapshotArchiveCleanupPendingError, type SnapshotArchiveCleanupPending } from './SnapshotArchiveCleanup' +import { WERR_INVALID_OPERATION, WERR_UNAUTHORIZED } from '../../../sdk/WERR_errors' +import type { StorageKnex } from '../../StorageKnex' +import { KnexSnapshotArchiveService } from './KnexSnapshotArchiveService' +import { readSnapshotArchiveSourceSchema } from './KnexSnapshotArchiveSource' +import { snapshotArchiveDatabaseNow } from './SnapshotArchiveSql' +import { + parseSnapshotArchiveRpcInput, + snapshotArchiveCapabilities, + type SnapshotArchiveMethod, + type SnapshotArchiveOffer +} from './SnapshotArchiveProtocol' + +/** Authenticated dispatch. The HTTP edge still owns framing, rate and body limits. */ +export class KnexSnapshotArchiveRpc { + private readonly service: KnexSnapshotArchiveService + private stopped = false + + constructor(private readonly storage: StorageKnex) { + this.service = new KnexSnapshotArchiveService(storage) + } + + async capabilities() { + if ( + this.stopped || + (this.storage.chain !== 'main' && this.storage.chain !== 'test') || + this.storage.getSnapshotSync() === undefined || + !(await this.storage.supportsSnapshotArchiveSource()) + ) + return undefined + for (const name of [ + 'snapshot_archive_requests', + 'snapshot_archive_owners', + 'snapshot_archive_owner_slots', + 'snapshot_archives', + 'snapshot_archive_pages', + 'snapshot_archive_capacity' + ]) { + if (!(await this.storage.knex.schema.hasTable(name))) return undefined + } + return this.stopped ? undefined : snapshotArchiveCapabilities + } + + private async sourceOffer(serverTime?: number): Promise { + const chain = this.storage.chain + if (chain !== 'main' && chain !== 'test') throw new WERR_INVALID_OPERATION('Snapshot archive chain is unavailable') + const sourceSchema = await readSnapshotArchiveSourceSchema(this.storage, this.storage.knex) + const settings = this.storage.getSettings() + return { + version: 1, + sourceStorageIdentityKey: settings.storageIdentityKey, + sourceSchema, + chain, + serverTime: serverTime ?? (await snapshotArchiveDatabaseNow(this.storage.knex)) + } + } + + async dispatch(method: SnapshotArchiveMethod, params: unknown[], authenticatedIdentityKey: string): Promise { + const input = parseSnapshotArchiveRpcInput(method, params) + if (input.identityKey !== authenticatedIdentityKey) + throw new WERR_UNAUTHORIZED('Snapshot archive identity must match authentication') + if ((await this.capabilities()) === undefined) + throw new WERR_INVALID_OPERATION('Snapshot archive transport is unavailable') + const identityKey = authenticatedIdentityKey + switch (input.method) { + case 'getSnapshotArchiveReaderOffer': { + // Read source metadata before issuing a retained request, so failed metadata + // cannot leave a caller-owned request that was never returned. + const offer = await this.sourceOffer() + const issued = await this.service.offerReader(identityKey, input.options) + if (issued === undefined) + return { version: 1, outcome: 'resource-limited' } satisfies SnapshotArchiveReaderOffer + return { + version: 1, + outcome: 'offered', + offer: { ...offer, serverTime: issued.serverTime }, + request: issued.request + } satisfies SnapshotArchiveReaderOffer + } + case 'admitSnapshotArchive': + return await this.service.admitReader(identityKey, input.request) + case 'cancelSnapshotArchiveRequest': + try { + await this.service.cancelReader(identityKey, input.request) + return true + } catch (error) { + if (!(error instanceof SnapshotArchiveCleanupPendingError)) throw error + return { + version: 1, + outcome: 'cleanup-pending', + requestId: input.request.requestId + } satisfies SnapshotArchiveCleanupPending + } + case 'getSnapshotArchiveOffer': + return await this.sourceOffer() + case 'startSnapshotArchive': + return await this.service.start(identityKey, input.request) + case 'getSnapshotArchiveStatus': + return await this.service.status(identityKey, input.requestId) + case 'getSnapshotArchiveDirectory': + return await this.service.directory(identityKey, input.archiveId) + case 'readSnapshotArchivePage': + return await this.service.read(identityKey, input.archiveId, input.sequence) + case 'cancelSnapshotArchive': + await this.service.cancel(identityKey, input.requestId) + return true + } + } + + close(): Promise { + this.stopped = true + return this.service.close() + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveService.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveService.test.ts new file mode 100644 index 000000000..89db38ce8 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveService.test.ts @@ -0,0 +1,901 @@ +import { SnapshotArchiveCleanupPendingError } from './SnapshotArchiveOwner' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { seedArchiveClosure } from '../../../../test/utils/snapshotArchiveFixtures' +import { KnexSnapshotArchiveService } from './KnexSnapshotArchiveService' +import { KnexSnapshotArchiveRequestStore } from './KnexSnapshotArchiveRequestStore' +import { KnexSnapshotArchiveStore } from './KnexSnapshotArchiveStore' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import { verifySnapshotArchiveDirectory, verifySnapshotArchivePage } from './SnapshotArchiveDirectory' +import * as ArchiveSql from './SnapshotArchiveSql' +import * as ArchiveGuard from './SnapshotArchiveGuard' + +const identity = '02' + '11'.repeat(32) +const other = '03' + '22'.repeat(32) +const stores: StorageKnex[] = [] +const services: KnexSnapshotArchiveService[] = [] +const directories: string[] = [] +function gate() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} +function service(storage: StorageKnex) { + const controller = new KnexSnapshotArchiveService(storage) + services.push(controller) + return controller +} +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-service-')) + directories.push(directory) + const open = () => { + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) + stores.push(storage) + return storage + } + const storage = open() + await storage.knex.raw('PRAGMA journal_mode = WAL') + await storage.migrate('service source', 'service-source') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identity) + const { user: peer } = await storage.findOrInsertUser(other) + await seedArchiveClosure(storage, user.userId, peer.userId) + return { storage, controller: service(storage), open } +} +async function closeRepairedSource(storage: StorageKnex, requestId: string): Promise { + const requests = new KnexSnapshotArchiveRequestStore(storage.knex) + await expect(requests.close(identity, requestId, 'failed')).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + const owner = await storage.knex('snapshot_archive_owners').where({ identityKey: identity, requestId }).first() + expect(owner).toBeDefined() + await requests.sourceClosed({ identityKey: identity, requestId, claimToken: owner.claimToken }) + await requests.close(identity, requestId, 'failed') + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(0) +} +function request(nonce = 'b'.repeat(64)) { + const fields = { version: 1 as const, nonce, notAfter: Date.now() + 300000, maxBytes: 32768 } + return { ...fields, requestId: snapshotArchiveRequestId(fields) } +} +afterEach(async () => { + jest.restoreAllMocks() + await Promise.all(services.splice(0).map(controller => controller.close().catch(() => undefined))) + await Promise.all(stores.splice(0).map(storage => storage.destroy())) + await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))) +}) + +test('capture reserves before reader acquisition and a replacement server recovers the exact immutable archive', async () => { + const { storage, controller, open } = await fixture() + const input = request() + const original = storage.openSnapshotArchiveSource.bind(storage) + const opening = jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementation(async (key, options, owner) => { + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + expect((await controller.status(identity, input.requestId)).state).toBe('building') + expect(options?.signal).toBeInstanceOf(AbortSignal) + expect(options?.lifetimeMs).toBeGreaterThan(0) + expect(options?.lifetimeMs).toBeLessThanOrEqual(300000) + return await original(key, options, owner) + }) + const pending = controller.create(identity, input) + expect(controller.create(identity, { ...input })).toBe(pending) + expect(() => controller.create(other, input)).toThrow('opening or active') + expect(() => controller.create(identity, request('c'.repeat(64)))).toThrow('opening or active') + const ready = await pending + expect(ready).toMatchObject({ state: 'ready', requestId: input.requestId, expiresAt: input.notAfter }) + expect(opening).toHaveBeenCalledTimes(1) + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + const directory = await controller.directory(identity, ready.archiveId!) + const verified = verifySnapshotArchiveDirectory(directory, { + identityKey: identity, + chain: 'test', + sourceStorageIdentityKey: 'service-source', + digest: ready.digest + }) + expect(verified.manifest.pages).toBe(13) + const page = await controller.read(identity, ready.archiveId!, 8) + expect(verifySnapshotArchivePage(page, verified.receipts[8])).toEqual(page.bytes) + await expect(controller.directory(other, ready.archiveId!)).rejects.toThrow('unavailable') + await expect(controller.read(other, ready.archiveId!, 8)).rejects.toThrow('unavailable') + await expect(controller.status(other, input.requestId)).rejects.toThrow('unavailable') + await controller.cancel(other, input.requestId) + expect(await controller.status(identity, input.requestId)).toEqual(ready) + await controller.close() + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + const replacementOpen = jest.spyOn(replacementStorage, 'openSnapshotArchiveSource') + expect(await replacement.create(identity, input)).toEqual(ready) + expect(replacementOpen).not.toHaveBeenCalled() + expect(await replacement.directory(identity, ready.archiveId!)).toEqual(directory) + expect(await replacement.read(identity, ready.archiveId!, 8)).toEqual(page) + await replacement.cancel(identity, input.requestId) + expect((await replacement.create(identity, input)).state).toBe('closed') + expect(replacementOpen).not.toHaveBeenCalled() + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test.each(['cancel', 'shutdown'] as const)( + '%s during reader opening waits for physical destruction before releasing quota', + async action => { + const { storage, controller } = await fixture() + const input = request() + const entered = gate() + const allowOpen = gate() + const destroying = gate() + const allowDestroy = gate() + const original = storage.openSnapshotArchiveSource.bind(storage) + jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementation(async (key, options, owner) => { + // Acquire the actual view, but withhold it from the controller until cancellation. + const source = (await original(key, { ...options, signal: undefined }, owner))! + const reader = Reflect.get(storage, 'snapshotSyncSource') as StorageKnex + const destroy = reader.destroy.bind(reader) + jest.spyOn(reader, 'destroy').mockImplementation(async () => { + destroying.resolve() + await allowDestroy.promise + await destroy() + }) + entered.resolve() + await allowOpen.promise + return source + }) + const capture = controller.create(identity, input) + void capture.catch(() => undefined) + let stopping: Promise | undefined + try { + await entered.promise + let finished = false + stopping = (action === 'cancel' ? controller.cancel(identity, input.requestId) : controller.close()).then(() => { + finished = true + }) + void stopping.catch(() => undefined) + await Promise.resolve() + expect(finished).toBe(false) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + allowOpen.resolve() + await destroying.promise + expect(finished).toBe(false) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + await storage.knex('tx_labels').where({ txLabelId: 1 }).update({ label: 'foreground during cancellation' }) + allowDestroy.resolve() + await stopping + await expect(capture).rejects.toThrow('cancelled') + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + expect((await new KnexSnapshotArchiveRequestStore(storage.knex).status(identity, input.requestId)).state).toBe( + 'closed' + ) + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await storage.knex('snapshot_archive_pages')).toHaveLength(0) + if (action === 'cancel') { + expect((await controller.create(identity, input)).state).toBe('closed') + jest.restoreAllMocks() + expect((await controller.create(identity, request('c'.repeat(64)))).state).toBe('ready') + } else { + expect(() => controller.create(identity, request('c'.repeat(64)))).toThrow('closed') + } + } finally { + allowOpen.resolve() + allowDestroy.resolve() + await capture.catch(() => undefined) + await stopping?.catch(() => undefined) + } + } +) + +test('shutdown before the admission microtask prevents a claim and remains idempotent', async () => { + const { storage, controller } = await fixture() + const input = request() + const capture = controller.create(identity, input) + const closing = controller.close() + expect(controller.close()).toBe(closing) + await closing + await expect(capture).rejects.toThrow('cancelled') + expect(await storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) + expect(() => controller.create(identity, input)).toThrow('closed') + await expect(controller.status(identity, input.requestId)).rejects.toThrow('closed') + await expect(controller.directory(identity, 'a'.repeat(64))).rejects.toThrow('closed') + await expect(controller.read(identity, 'a'.repeat(64), 0)).rejects.toThrow('closed') + await expect(controller.cancel(identity, input.requestId)).rejects.toThrow('closed') +}) + +test('idle shutdown preserves ready archives and closing never destroys the foreground provider', async () => { + const { storage, controller } = await fixture() + const input = request() + const ready = await controller.create(identity, input) + const closing = controller.close() + expect(controller.close()).toBe(closing) + await closing + const replacement = service(storage) + expect(await replacement.status(identity, input.requestId)).toEqual(ready) + expect((await replacement.directory(identity, ready.archiveId!)).archiveId).toBe(ready.archiveId) + expect(await storage.knex('users')).toHaveLength(2) +}) + +test('capture failure closes the reader, records a terminal failure and allows a different request', async () => { + const { storage, controller } = await fixture() + const input = request() + const original = storage.openSnapshotArchiveSource.bind(storage) + const opening = jest + .spyOn(storage, 'openSnapshotArchiveSource') + .mockImplementationOnce(async (key, options, owner) => { + const source = (await original(key, options, owner))! + return { + ...source, + validateClosure: async () => { + throw new Error('fixture closure failure') + } + } + }) + await expect(controller.create(identity, input)).rejects.toThrow('fixture closure failure') + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + expect((await controller.create(identity, input)).state).toBe('failed') + expect(opening).toHaveBeenCalledTimes(1) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) + expect((await controller.create(identity, request('c'.repeat(64)))).state).toBe('ready') +}) + +test('failed physical cleanup retains the reservation and fences the controller including later shutdown', async () => { + const { storage, controller } = await fixture() + const input = request() + const original = storage.openSnapshotArchiveSource.bind(storage) + const failure = new Error('physical close failed') + let source: Awaited> + jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementationOnce(async (key, options, owner) => { + source = (await original(key, options, owner))! + return { + ...source, + close: async () => { + throw failure + } + } + }) + await expect(controller.create(identity, input)).rejects.toBe(failure) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + expect((await storage.knex('snapshot_archives').first()).state).toBe('building') + expect(() => controller.create(identity, request('c'.repeat(64)))).toThrow('closed') + await expect(controller.close()).rejects.toBe(failure) + await source!.close() + await closeRepairedSource(storage, input.requestId) +}) + +test('unsupported and busy sources fail without disturbing an existing local source', async () => { + const { storage, controller } = await fixture() + const local = (await storage.getSnapshotSync()!.openSource(identity))! + const input = request() + await expect(controller.create(identity, input)).rejects.toThrow('opening or active') + expect(local.isOpen).toBe(true) + expect((await controller.status(identity, input.requestId)).state).toBe('failed') + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) + await local.close() + jest.spyOn(storage, 'openSnapshotArchiveSource').mockResolvedValueOnce(undefined) + const unsupported = request('c'.repeat(64)) + await expect(controller.create(identity, unsupported)).rejects.toThrow('requires SQLite WAL or MySQL') + expect((await controller.status(identity, unsupported.requestId)).state).toBe('failed') + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) +}) + +test('an already claimed request is not resumed under a replacement source', async () => { + const { storage, controller } = await fixture() + const input = request() + const claimed = await new KnexSnapshotArchiveRequestStore(storage.knex).claim(identity, input) + const open = jest.spyOn(storage, 'openSnapshotArchiveSource') + expect(await controller.create(identity, input)).toEqual(claimed.receipt) + expect(open).not.toHaveBeenCalled() + await controller.cancel(identity, input.requestId) + expect((await controller.create(identity, input)).state).toBe('closed') +}) + +test('expiry between durable admission and source acquisition refuses without opening a pool', async () => { + const { storage, controller } = await fixture() + const input = request() + const claim = KnexSnapshotArchiveRequestStore.prototype.claim + jest.spyOn(KnexSnapshotArchiveRequestStore.prototype, 'claim').mockImplementationOnce(async function ( + this: KnexSnapshotArchiveRequestStore, + key, + value + ) { + const admitted = await claim.call(this, key, value) + jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(input.notAfter) + return admitted + }) + const open = jest.spyOn(storage, 'openSnapshotArchiveSource') + await expect(controller.create(identity, input)).rejects.toThrow('expired before capture') + expect(open).not.toHaveBeenCalled() + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) +}) + +test('invalid requests cannot occupy the local slot or shared quota', async () => { + const { storage, controller } = await fixture() + expect(() => controller.create(identity, { ...request(), writerToken: 'untrusted' })).toThrow('Invalid') + await expect(controller.create('not-a-profile', request())).rejects.toThrow('identityKey') + expect(await storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect((await controller.create(identity, request())).state).toBe('ready') +}) + +test('failed cleanup while opening retains admission even though no source was returned', async () => { + const { storage, controller } = await fixture() + const input = request() + const failure = new Error('owned pool destruction failed') + const destroy = StorageKnex.prototype.destroy + jest + .spyOn(ArchiveGuard, 'readGuardedSnapshotArchive') + .mockRejectedValueOnce(new Error('reader initialization failed')) + jest.spyOn(StorageKnex.prototype, 'destroy').mockImplementation(async function (this: StorageKnex) { + if (this !== storage) throw failure + await destroy.call(this) + }) + let received: unknown + try { + await controller.create(identity, input) + } catch (error) { + received = error + } + expect(received).toMatchObject({ name: 'SnapshotArchiveSourceCleanupError', cause: failure }) + expect((received as Error).message).toBe('Snapshot archive source cleanup failed') + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + expect(await storage.knex('snapshot_archives')).toHaveLength(0) + expect((await storage.knex('snapshot_archive_requests').first()).state).toBe('claimed') + expect(await storage.supportsSnapshotArchiveSource()).toBe(false) + expect(() => controller.create(identity, request('c'.repeat(64)))).toThrow('closed') + await expect(controller.close()).rejects.toBe(received) + jest.restoreAllMocks() + const reader = Reflect.get(storage, 'snapshotSyncSource') as StorageKnex + await reader.destroy() + await closeRepairedSource(storage, input.requestId) +}) + +test('start returns a durable receipt before capture completes and repeats only that admission', async () => { + const { storage, controller, open } = await fixture() + const input = request() + const entered = gate() + const finish = gate() + const original = storage.openSnapshotArchiveSource.bind(storage) + jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementationOnce(async (key, options, owner) => { + entered.resolve() + await finish.promise + return await original(key, options, owner) + }) + const accepted = controller.start(identity, input) + expect(controller.start(identity, { ...input })).toBe(accepted) + const completion = controller.create(identity, input) + void completion.catch(() => undefined) + try { + const receipt = await accepted + expect(receipt).toEqual({ version: 1, state: 'building', requestId: input.requestId, expiresAt: input.notAfter }) + expect(Object.isFrozen(receipt)).toBe(true) + await entered.promise + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + expect(await replacement.start(identity, input)).toEqual(receipt) + expect(Reflect.get(replacementStorage, 'snapshotSyncSource')).toBeUndefined() + finish.resolve() + const ready = await completion + expect(ready.state).toBe('ready') + expect(await replacement.start(identity, input)).toEqual(ready) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + await replacement.cancel(identity, input.requestId) + } finally { + finish.resolve() + await completion.catch(() => undefined) + } +}) + +test('resource exhaustion has a durable distinct terminal status and never replaces its archive on retry', async () => { + const { storage, controller, open } = await fixture() + const fields = { ...request(), maxBytes: 4097 } + const input = { ...fields, requestId: snapshotArchiveRequestId(fields) } + await expect(controller.create(identity, input)).rejects.toBeInstanceOf(SnapshotResourceLimitError) + expect(await controller.status(identity, input.requestId)).toEqual({ + version: 1, + requestId: input.requestId, + expiresAt: input.notAfter, + state: 'resource-limited' + }) + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + const reader = jest.spyOn(replacementStorage, 'openSnapshotArchiveSource') + expect((await replacement.start(identity, input)).state).toBe('resource-limited') + expect(reader).not.toHaveBeenCalled() + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await replacement.cancel(identity, input.requestId) + expect((await replacement.status(identity, input.requestId)).state).toBe('resource-limited') + expect((await replacement.create(identity, request('c'.repeat(64)))).state).toBe('ready') +}) + +test.each(['pending', 'capturing', 'ready', 'legacy'] as const)( + 'new admission reaps an abandoned expired %s capture', + async state => { + const { storage, controller } = await fixture() + const input = request() + const requests = new KnexSnapshotArchiveRequestStore(storage.knex) + const archives = new KnexSnapshotArchiveStore(storage.knex) + if (state === 'ready') await controller.create(identity, input) + else { + const admitted = state === 'legacy' ? undefined : await requests.claim(identity, input) + if (state === 'capturing' || state === 'legacy') { + const source = (await storage.openSnapshotArchiveSource(identity))! + const binding = { + version: 1 as const, + snapshotId: source.snapshotId, + sourceSchema: source.sourceSchema, + sourceStorage: source.sourceStorage, + user: source.user + } + await source.close() + const writer = + state === 'legacy' + ? await archives.begin(binding, { maxBytes: 32768, lifetimeMs: 200000 }) + : await requests.begin(admitted!.owner!, binding) + await archives.append(writer, { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes: Uint8Array.of(0) }) + } + } + const now = input.notAfter + 60000 + jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(now) + const fields = { ...input, nonce: 'c'.repeat(64), notAfter: now + 300000 } + const next = { ...fields, requestId: snapshotArchiveRequestId(fields) } + const ready = await controller.create(identity, next) + expect(ready.state).toBe('ready') + expect(await storage.knex('snapshot_archive_requests')).toHaveLength(1) + expect(await storage.knex('snapshot_archives')).toHaveLength(1) + expect((await storage.knex('snapshot_archives').first()).archiveId).toBe(ready.archiveId) + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 1, + reservedBytes: next.maxBytes + }) + await controller.cancel(identity, next.requestId) + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + } +) + +test('a second controller fences a first start delayed before its durable claim', async () => { + const { storage, controller, open } = await fixture() + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + const input = request() + const entered = gate() + const resume = gate() + const original = KnexSnapshotArchiveRequestStore.prototype.claim + jest.spyOn(KnexSnapshotArchiveRequestStore.prototype, 'claim').mockImplementation(async function ( + this: KnexSnapshotArchiveRequestStore, + key, + value + ) { + entered.resolve() + await resume.promise + return await original.call(this, key, value) + }) + const opening = jest.spyOn(storage, 'openSnapshotArchiveSource') + const pending = controller.create(identity, input) + void pending.catch(() => undefined) + try { + await entered.promise + // Client-side abort alone cannot drain this delayed server operation. + await replacement.cancelRequest(identity, input) + await replacement.cancelRequest(identity, input) + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + resume.resolve() + expect((await pending).state).toBe('closed') + expect(opening).not.toHaveBeenCalled() + expect(await storage.knex('snapshot_archive_pages')).toHaveLength(0) + expect((await replacement.create(identity, input)).state).toBe('closed') + } finally { + resume.resolve() + await pending.catch(() => undefined) + } +}) + +test('reader offers acquire no source and a replacement recovers only the admitted immutable archive', async () => { + const { storage, controller, open } = await fixture() + const opening = jest.spyOn(storage, 'openSnapshotArchiveSource') + const offered = (await controller.offerReader(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + expect(offered.request.version).toBe(2) + expect(opening).not.toHaveBeenCalled() + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await expect(controller.admitReader(other, offered.request)).rejects.toThrow('unavailable') + const admitted = await controller.admitReader(identity, offered.request) + expect(admitted).toMatchObject({ outcome: 'accepted', receipt: { state: 'building' } }) + let receipt = await controller.status(identity, offered.request.requestId) + for (let attempt = 0; receipt.state === 'building' && attempt < 100; attempt++) { + await new Promise(resolve => setTimeout(resolve, 10)) + receipt = await controller.status(identity, offered.request.requestId) + } + expect(receipt.state).toBe('ready') + expect(opening).toHaveBeenCalledTimes(1) + expect(Reflect.get(storage, 'snapshotSyncSource')).toBeUndefined() + const directory = await controller.directory(identity, receipt.archiveId!) + await controller.close() + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + const replacementOpening = jest.spyOn(replacementStorage, 'openSnapshotArchiveSource') + expect(await replacement.admitReader(identity, offered.request)).toEqual({ + version: 1, + outcome: 'accepted', + receipt + }) + expect(await replacement.directory(identity, receipt.archiveId!)).toEqual(directory) + expect(replacementOpening).not.toHaveBeenCalled() + await replacement.cancelReader(identity, offered.request) + await replacement.cancelReader(identity, offered.request) + expect(await storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect(await storage.knex('snapshot_archive_pages')).toHaveLength(0) + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await expect(replacement.admitReader(identity, offered.request)).rejects.toThrow('unavailable') + expect(replacementOpening).not.toHaveBeenCalled() +}) + +test('reader cancellation on a replacement removes an offer before a delayed admission can capture', async () => { + const { storage, controller, open } = await fixture() + const offered = (await controller.offerReader(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + const entered = gate() + const resume = gate() + const original = KnexSnapshotArchiveRequestStore.prototype.claimReader + jest.spyOn(KnexSnapshotArchiveRequestStore.prototype, 'claimReader').mockImplementation(async function ( + this: KnexSnapshotArchiveRequestStore, + key, + value + ) { + entered.resolve() + await resume.promise + return await original.call(this, key, value) + }) + const opening = jest.spyOn(storage, 'openSnapshotArchiveSource') + const pending = controller.admitReader(identity, offered.request) + void pending.catch(() => undefined) + try { + await entered.promise + await replacement.cancelReader(identity, offered.request) + await replacement.cancelReader(identity, offered.request) + expect(await storage.knex('snapshot_archive_requests')).toHaveLength(0) + resume.resolve() + await expect(pending).rejects.toThrow('unavailable') + expect(opening).not.toHaveBeenCalled() + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await storage.knex('snapshot_archive_pages')).toHaveLength(0) + await expect(replacement.admitReader(identity, offered.request)).rejects.toThrow('unavailable') + expect(opening).not.toHaveBeenCalled() + } finally { + resume.resolve() + await pending.catch(() => undefined) + } +}) + +test.each(['failed', 'resource-limited'] as const)( + 'reader %s capture remains observable until explicit cancellation collects its receipt', + async state => { + const { storage, controller, open } = await fixture() + const offered = (await controller.offerReader(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! + const failure = state === 'failed' ? new Error('synthetic capture failure') : new SnapshotResourceLimitError('full') + const opening = jest.spyOn(storage, 'openSnapshotArchiveSource').mockRejectedValueOnce(failure) + expect(await controller.admitReader(identity, offered.request)).toMatchObject({ outcome: 'accepted' }) + let receipt = await controller.status(identity, offered.request.requestId) + for (let attempt = 0; receipt.state === 'building' && attempt < 100; attempt++) { + await new Promise(resolve => setTimeout(resolve, 10)) + receipt = await controller.status(identity, offered.request.requestId) + } + expect(receipt.state).toBe(state) + await controller.close() + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + expect(await replacement.admitReader(identity, offered.request)).toEqual({ + version: 1, + outcome: 'accepted', + receipt + }) + expect(opening).toHaveBeenCalledTimes(1) + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await replacement.cancelReader(identity, offered.request) + expect(await storage.knex('snapshot_archive_requests')).toHaveLength(0) + await expect(replacement.admitReader(identity, offered.request)).rejects.toThrow('unavailable') + } +) + +test('cancellation through another controller retains quota until the capturing owner drains its physical pool', async () => { + const { storage, controller, open } = await fixture() + const replacementStorage = open() + await replacementStorage.makeAvailable() + const replacement = service(replacementStorage) + const input = request() + const reading = gate() + const allowRead = gate() + const destroying = gate() + const allowDestroy = gate() + const original = storage.openSnapshotArchiveSource.bind(storage) + jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementationOnce(async (key, options, owner) => { + const source = (await original(key, options, owner))! + const reader = Reflect.get(storage, 'snapshotSyncSource') as StorageKnex + const destroy = reader.knex.client.destroyRawConnection.bind(reader.knex.client) + jest.spyOn(reader.knex.client, 'destroyRawConnection').mockImplementation(async connection => { + destroying.resolve() + await allowDestroy.promise + await destroy(connection) + }) + return { + ...source, + readPage: async (...args) => { + reading.resolve() + await allowRead.promise + return await source.readPage(...args) + } + } + }) + const capture = controller.create(identity, input) + void capture.catch(() => undefined) + try { + await reading.promise + await expect(replacement.cancelRequest(identity, input)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect((await replacement.status(identity, input.requestId)).state).toBe('closed') + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(1) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + allowRead.resolve() + await destroying.promise + await expect(replacement.cancelRequest(identity, input)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect(await storage.knex('snapshot_archive_pages')).toHaveLength(0) + await storage.knex('tx_labels').where({ txLabelId: 1 }).update({ label: 'foreground while owner drains' }) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + allowDestroy.resolve() + await expect(capture).rejects.toThrow('unavailable') + await replacement.cancelRequest(identity, input) + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(0) + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + } finally { + allowRead.resolve() + allowDestroy.resolve() + await capture.catch(() => undefined) + } +}) + +test('admission envelopes distinguish acceptance, transient busy capacity and an active reader offer', async () => { + const { storage, controller } = await fixture() + const offered = (await controller.offerReader(other, { lifetimeMs: 300000, maxBytes: 32768 }))! + const input = request() + const alternative = request('c'.repeat(64)) + const entered = gate() + const resume = gate() + const original = storage.openSnapshotArchiveSource.bind(storage) + jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementationOnce(async (...args) => { + entered.resolve() + await resume.promise + return await original(...args) + }) + const capture = controller.create(identity, input) + void capture.catch(() => undefined) + try { + expect(await controller.admitRequest(identity, input)).toEqual({ + version: 1, + outcome: 'accepted', + receipt: { version: 1, requestId: input.requestId, state: 'building', expiresAt: input.notAfter } + }) + await entered.promise + expect(await controller.offerReader(other, { lifetimeMs: 300000, maxBytes: 32768 })).toBeUndefined() + expect(await controller.admitRequest(other, alternative)).toEqual({ + version: 1, + outcome: 'resource-limited', + requestId: alternative.requestId, + expiresAt: alternative.notAfter + }) + expect(await controller.admitReader(other, offered.request)).toEqual({ + version: 1, + outcome: 'resource-limited', + requestId: offered.request.requestId, + expiresAt: offered.request.notAfter + }) + resume.resolve() + expect((await capture).state).toBe('ready') + await controller.cancel(identity, input.requestId) + await controller.cancelReader(other, offered.request) + } finally { + resume.resolve() + await capture.catch(() => undefined) + } +}) + +test('admission never converts an independent recovery error or invalid input into capacity refusal', async () => { + const { storage, controller } = await fixture() + const failure = new Error('synthetic independent recovery failure') + jest.spyOn(storage, 'recoverSnapshotArchiveSources').mockRejectedValueOnce(failure) + await expect(controller.admitRequest(identity, request())).rejects.toBe(failure) + await expect(controller.admitRequest(identity, { ...request(), maxBytes: 0 })).rejects.toThrow('Invalid') + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) +}) + +test.each([true, false])('shutdown drains the result of a delayed offer, available=%s', async available => { + const { storage, controller } = await fixture() + const entered = gate() + const resume = gate() + const original = KnexSnapshotArchiveRequestStore.prototype.offer + jest.spyOn(KnexSnapshotArchiveRequestStore.prototype, 'offer').mockImplementationOnce(async function ( + this: KnexSnapshotArchiveRequestStore, + ...args + ) { + const offer = available ? await original.apply(this, args) : undefined + entered.resolve() + await resume.promise + return offer + }) + const pending = controller.offerReader(identity, { lifetimeMs: 300000, maxBytes: 32768 }) + void pending.catch(() => undefined) + try { + await entered.promise + await controller.close() + resume.resolve() + await expect(pending).rejects.toThrow('service is closed') + expect(await storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) + } finally { + resume.resolve() + await pending.catch(() => undefined) + } +}) + +test.each(['creation', 'reader'] as const)( + 'exact %s cancellation drains its active capture without cancelling a different request', + async kind => { + const { storage, controller } = await fixture() + const offered = + kind === 'reader' ? (await controller.offerReader(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! : undefined + const alternative = + kind === 'reader' ? (await controller.offerReader(identity, { lifetimeMs: 300000, maxBytes: 32768 }))! : undefined + const input = offered?.request ?? request() + const otherInput = alternative?.request ?? request('c'.repeat(64)) + const reading = gate() + const resume = gate() + const aborted = gate() + let signal: AbortSignal | undefined + const original = storage.openSnapshotArchiveSource.bind(storage) + jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementationOnce(async (key, options, owner) => { + signal = options?.signal + signal!.addEventListener('abort', aborted.resolve, { once: true }) + const source = (await original(key, options, owner))! + return { + ...source, + readPage: async (...args) => { + reading.resolve() + await resume.promise + return await source.readPage(...args) + } + } + }) + const cancel = (value: unknown) => + kind === 'reader' ? controller.cancelReader(identity, value) : controller.cancelRequest(identity, value) + let cancellation: Promise | undefined + try { + if (kind === 'reader') await controller.admitReader(identity, input) + else await controller.start(identity, input) + await reading.promise + await cancel(otherInput) + expect(signal?.aborted).toBe(false) + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(1) + let settled = false + cancellation = cancel(input).finally(() => { + settled = true + }) + void cancellation.catch(() => undefined) + await aborted.promise + expect(settled).toBe(false) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + resume.resolve() + await cancellation + expect(await storage.knex('snapshot_archive_owners')).toHaveLength(0) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) + if (kind === 'reader') expect(await storage.knex('snapshot_archive_requests')).toHaveLength(0) + else expect((await controller.status(identity, input.requestId)).state).toBe('closed') + } finally { + resume.resolve() + await cancellation?.catch(() => undefined) + } + } +) + +test('idle service shutdown waits for an already admitted provider recovery without destroying foreground storage', async () => { + const { storage, controller } = await fixture() + const entered = gate() + const resume = gate() + const original = ArchiveGuard.recoverSnapshotArchiveGuards + jest.spyOn(ArchiveGuard, 'recoverSnapshotArchiveGuards').mockImplementationOnce(async (...args) => { + entered.resolve() + await resume.promise + await original(...args) + }) + const recovery = storage.recoverSnapshotArchiveSources() + void recovery.catch(() => undefined) + let closing: Promise | undefined + try { + await entered.promise + let settled = false + closing = controller.close().finally(() => { + settled = true + }) + void closing.catch(() => undefined) + await new Promise(resolve => setImmediate(resolve)) + expect(settled).toBe(false) + resume.resolve() + await Promise.all([recovery, closing]) + expect((await storage.findUsers({ partial: { identityKey: identity } }))[0].identityKey).toBe(identity) + } finally { + resume.resolve() + await Promise.allSettled([recovery, closing]) + } +}) + +test.each([1, 1234, 600000])('source lifetime respects exactly %s remaining database milliseconds', async remaining => { + const { storage, controller } = await fixture() + const input = request() + jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(input.notAfter - remaining) + const opening = jest.spyOn(storage, 'openSnapshotArchiveSource').mockResolvedValueOnce(undefined) + await expect(controller.create(identity, input)).rejects.toThrow('requires SQLite WAL or MySQL') + expect(opening).toHaveBeenCalledWith( + identity, + { + signal: expect.any(AbortSignal), + lifetimeMs: Math.min(300000, remaining) + }, + expect.objectContaining({ identityKey: identity, requestId: input.requestId }) + ) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(0) +}) + +test('cancelling an active capture reports its cleanup failure and retains the reservation', async () => { + const { storage, controller } = await fixture() + const input = request() + const reading = gate() + const resume = gate() + const aborted = gate() + const failure = new Error('synthetic capture cleanup failure') + const original = storage.openSnapshotArchiveSource.bind(storage) + let source: Awaited> + jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementationOnce(async (key, options, owner) => { + options!.signal!.addEventListener('abort', aborted.resolve, { once: true }) + source = (await original(key, options, owner))! + return { + ...source, + readPage: async (...args) => { + reading.resolve() + await resume.promise + return await source!.readPage(...args) + }, + close: async () => { + throw failure + } + } + }) + const capture = controller.create(identity, input) + void capture.catch(() => undefined) + let cancellation: Promise | undefined + try { + await reading.promise + cancellation = controller.cancelRequest(identity, input) + void cancellation.catch(() => undefined) + await aborted.promise + resume.resolve() + await expect(capture).rejects.toBe(failure) + await expect(cancellation).rejects.toBe(failure) + expect((await storage.knex('snapshot_archive_capacity').first()).archives).toBe(1) + await expect(controller.close()).rejects.toBe(failure) + } finally { + resume.resolve() + await Promise.allSettled([capture, cancellation]) + await source?.close() + await closeRepairedSource(storage, input.requestId) + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveService.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveService.ts new file mode 100644 index 000000000..110bcadfb --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveService.ts @@ -0,0 +1,264 @@ +import { parseSnapshotArchiveReaderRequest, type SnapshotArchiveReaderRequest } from './SnapshotArchiveReaderRequest' +import { validateSnapshotArchiveReaderOptions, type SnapshotArchiveReaderOptions } from './SnapshotArchiveReaderOffer' +import { WERR_INVALID_OPERATION, WERR_NOT_IMPLEMENTED } from '../../../sdk/WERR_errors' +import type { StorageKnex } from '../../StorageKnex' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { SnapshotArchiveAdmissionLimitError, type SnapshotArchiveAdmission } from './SnapshotArchiveAdmission' +import { SnapshotArchiveSourceCleanupError, type SnapshotArchiveSource } from './KnexSnapshotArchiveSource' +import { KnexSnapshotArchiveRequestStore } from './KnexSnapshotArchiveRequestStore' +import { KnexSnapshotArchiveStore } from './KnexSnapshotArchiveStore' +import { + parseSnapshotArchiveRequest, + type SnapshotArchiveRequest, + type SnapshotArchiveRequestReceipt, + type SnapshotArchiveRequestOwner, + type SnapshotArchiveTerminalState +} from './SnapshotArchiveRequest' +import { snapshotArchiveDatabaseNow } from './SnapshotArchiveSql' +import { assertSnapshotArchiveCaptureActive, captureSnapshotArchiveSource } from './captureSnapshotArchiveSource' + +interface Capture { + identityKey: string + request: Readonly + controller: AbortController + terminal: Exclude + accepted: Promise + completion: Promise +} + +/** Internal SQL controller. Authentication and bounded HTTP framing belong to its caller. */ +export class KnexSnapshotArchiveService { + private readonly requests: KnexSnapshotArchiveRequestStore + private readonly archives: KnexSnapshotArchiveStore + private active?: Capture + private stopped = false + private closing?: Promise + private cleanupFailure?: { error: unknown } + + constructor(private readonly storage: StorageKnex) { + this.requests = new KnexSnapshotArchiveRequestStore(storage.knex, true, true) + this.archives = new KnexSnapshotArchiveStore(storage.knex) + } + + private assertOpen(): void { + if (this.stopped) throw new WERR_INVALID_OPERATION('Snapshot archive service is closed') + } + + private failedCleanup(error: unknown): never { + this.cleanupFailure = { error } + this.stopped = true + throw error + } + + /** + * Admission is synchronous, before the first database or pool await. A retry + * keeps its fixed request ID; another process can recover its durable receipt. + * Completion waits for capture; status remains available while it runs. + */ + create(identityKey: string, input: unknown): Promise { + return this.admit(identityKey, input).completion + } + + /** Return the durable admission promptly; poll status while the owned capture continues. */ + start(identityKey: string, input: unknown): Promise { + return this.admit(identityKey, input).accepted + } + + /** Only a pre-admission capacity refusal selects the transient outcome. */ + async admitRequest(identityKey: string, input: unknown): Promise { + const request = parseSnapshotArchiveRequest(input) + try { + return { version: 1, outcome: 'accepted', receipt: await this.start(identityKey, request) } + } catch (error) { + if (!(error instanceof SnapshotArchiveAdmissionLimitError)) throw error + return { version: 1, outcome: 'resource-limited', requestId: request.requestId, expiresAt: request.notAfter } + } + } + + /** Offering retains metadata only; no SQL source or archive quota is acquired. */ + async offerReader(identityKey: string, input: SnapshotArchiveReaderOptions) { + this.assertOpen() + const options = validateSnapshotArchiveReaderOptions(input) + if (this.active !== undefined) return undefined + await this.storage.recoverSnapshotArchiveSources() + await this.requests.reap() + this.assertOpen() + await this.archives.reap() + this.assertOpen() + const offer = await this.requests.offer(identityKey, options) + if (this.stopped && offer !== undefined) await this.requests.close(identityKey, offer.request.requestId) + this.assertOpen() + return offer + } + + async admitReader(identityKey: string, input: unknown): Promise { + const request = parseSnapshotArchiveReaderRequest(input) + try { + return { version: 1, outcome: 'accepted', receipt: await this.admit(identityKey, request, true).accepted } + } catch (error) { + if (!(error instanceof SnapshotArchiveAdmissionLimitError)) throw error + return { version: 1, outcome: 'resource-limited', requestId: request.requestId, expiresAt: request.notAfter } + } + } + + private admit(identityKey: string, input: unknown, reader = false): Capture { + this.assertOpen() + const request = reader ? parseSnapshotArchiveReaderRequest(input) : parseSnapshotArchiveRequest(input) + if (this.active !== undefined) { + if (this.active.identityKey === identityKey && this.active.request.requestId === request.requestId) + return this.active + throw new SnapshotArchiveAdmissionLimitError('Snapshot archive capture is opening or active') + } + const claim = Promise.resolve().then(async () => { + assertSnapshotArchiveCaptureActive(job.controller.signal) + // Reap outside the claim transaction, before reserving or opening a pool. + await this.storage.recoverSnapshotArchiveSources() + await this.requests.reap() + await this.archives.reap() + assertSnapshotArchiveCaptureActive(job.controller.signal) + return reader + ? await this.requests.claimReader(identityKey, request) + : await this.requests.claim(identityKey, request) + }) + const accepted = claim.then(result => Object.freeze({ ...result.receipt })) + const completion = claim + .then(result => this.capture(job, result)) + .finally(() => { + if (this.active === job) this.active = undefined + }) + const job: Capture = { + identityKey, + request, + controller: new AbortController(), + terminal: 'failed', + accepted, + completion + } + this.active = job + // Cancellation/shutdown can observe completion even after a caller disconnects. + void completion.catch(() => undefined) + void accepted.catch(() => undefined) + return job + } + + private async capture( + job: Capture, + claimed: { receipt: SnapshotArchiveRequestReceipt; owner?: SnapshotArchiveRequestOwner } + ): Promise { + const { identityKey, request, controller } = job + if (claimed.owner === undefined) return claimed.receipt + const owner = claimed.owner + let source: SnapshotArchiveSource | undefined + const cleanup = async (error?: unknown): Promise => { + if (job.terminal !== 'closed' && error instanceof SnapshotResourceLimitError) job.terminal = 'resource-limited' + try { + // Keep the logical reservation through this process's physical cleanup. + await source?.close() + await this.requests.sourceClosed(owner) + await this.requests.close(identityKey, request.requestId, job.terminal) + } catch (error) { + this.failedCleanup(error) + } + } + try { + assertSnapshotArchiveCaptureActive(controller.signal) + const remaining = request.notAfter - (await snapshotArchiveDatabaseNow(this.storage.knex)) + if (remaining < 1) throw new SnapshotResourceLimitError('Snapshot archive request expired before capture') + source = await this.storage.openSnapshotArchiveSource( + identityKey, + { signal: controller.signal, lifetimeMs: Math.min(300000, remaining) }, + owner + ) + if (source === undefined) throw new WERR_NOT_IMPLEMENTED('Snapshot archive capture requires SQLite WAL or MySQL') + await captureSnapshotArchiveSource( + source, + { + begin: binding => this.requests.begin(owner, binding), + append: (writer, page) => this.requests.append(owner, writer, page), + seal: async writer => { + await this.requests.sourceClosed(owner) + return await this.requests.seal(owner, writer) + }, + close: (_identityKey, _archiveId, error) => cleanup(error) + }, + identityKey, + this.storage.chain, + { signal: controller.signal } + ) + return await this.requests.status(identityKey, request.requestId) + } catch (error) { + if (error instanceof SnapshotArchiveSourceCleanupError) this.failedCleanup(error) + await cleanup(error) + throw error + } + } + + async status(identityKey: string, requestId: string): Promise { + this.assertOpen() + return await this.requests.status(identityKey, requestId) + } + + async directory(identityKey: string, archiveId: string) { + this.assertOpen() + return await this.archives.directory(identityKey, archiveId) + } + + async read(identityKey: string, archiveId: string, sequence: number) { + this.assertOpen() + return await this.archives.read(identityKey, archiveId, sequence) + } + + private async stop(job: Capture): Promise { + job.terminal = 'closed' + job.controller.abort() + await job.completion.catch(() => undefined) + if (this.cleanupFailure !== undefined) throw this.cleanupFailure.error + } + + /** Fence the immutable request even if cancellation precedes its first claim. */ + async cancelRequest(identityKey: string, input: unknown): Promise { + this.assertOpen() + const request = parseSnapshotArchiveRequest(input) + await this.requests.markCancellation(identityKey, request) + const job = this.active + if (job?.identityKey === identityKey && job.request.requestId === request.requestId) await this.stop(job) + // A different replica can fence the request, but only proved source cleanup + // permits release. Its pending outcome must not be acknowledged as complete. + await this.storage.recoverSnapshotArchiveSources() + await this.requests.close(identityKey, request.requestId) + } + + async cancelReader(identityKey: string, input: unknown): Promise { + this.assertOpen() + const request = parseSnapshotArchiveReaderRequest(input) + await this.requests.markReaderCancellation(identityKey, request) + const job = this.active + if (job?.identityKey === identityKey && job.request.requestId === request.requestId) await this.stop(job) + await this.storage.recoverSnapshotArchiveSources() + await this.requests.close(identityKey, request.requestId) + } + + async cancel(identityKey: string, requestId: string): Promise { + this.assertOpen() + const job = this.active + if (job?.identityKey === identityKey && job.request.requestId === requestId) await this.stop(job) + await this.storage.recoverSnapshotArchiveSources() + await this.requests.close(identityKey, requestId) + } + + /** Fence admission and drain the owned capture; completed archives survive shutdown. */ + close(): Promise { + this.stopped = true + if (this.closing === undefined) { + const job = this.active + const capture = job === undefined ? this.closedWithoutCapture() : this.stop(job) + this.closing = capture.finally(() => this.storage.awaitSnapshotArchiveRecovery()) + } + return this.closing + } + + private closedWithoutCapture(): Promise { + if (this.cleanupFailure !== undefined) return Promise.reject(this.cleanupFailure.error) + return Promise.resolve() + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveSource.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveSource.ts new file mode 100644 index 000000000..3c11531d6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveSource.ts @@ -0,0 +1,129 @@ +import { readGenerationIndexState } from '../../schema/snapshotSqliteIndexState' +import { readSnapshotGlobalIndexState } from '../../schema/snapshotGlobalIndexMigration' +import { readSnapshotCertificateIndexState } from '../../schema/snapshotCertificateIndexMigration' +import { readSnapshotRelationIndexState } from '../../schema/snapshotRelationIndexMigration' +import { readSnapshotProfileIndexState } from '../../schema/snapshotProfileIndexMigration' +import { Random, Utils } from '@bsv/sdk' +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import type { StorageKnex } from '../../StorageKnex' +import { createKnexWalletSnapshotPageReader } from '../KnexWalletReadSnapshot' +import type { WalletReadSnapshot, WalletReadSnapshotOptions } from '../WalletReadSnapshot' +import { assertKnexSnapshotArchiveClosure } from './KnexSnapshotArchiveClosure' +import type { RetainedReadSnapshot } from '../RetainedReadSnapshot' + +/** Internal ownership failure; no caller may release its admission as cleaned up. */ +export class SnapshotArchiveSourceCleanupError extends Error { + constructor(override readonly cause: unknown) { + super('Snapshot archive source cleanup failed') + this.name = 'SnapshotArchiveSourceCleanupError' + } +} + +export interface SnapshotArchiveSource extends WalletReadSnapshot { + readonly sourceSchema: string + /** Verify profile relations using the same read view as the header and pages. */ + validateClosure: () => Promise +} + +export async function readSnapshotArchiveSourceSchema(storage: StorageKnex, k: Knex): Promise { + const config = storage.knex.client.config.migrations + const query = k(config?.tableName ?? 'knex_migrations') + .select('name') + .orderBy('id', 'desc') + if (config?.schemaName !== undefined) void query.withSchema(config.schemaName) + const row: { name?: unknown } | undefined = await query.first() + if (typeof row?.name !== 'string' || row.name.length < 1 || row.name.length > 256) { + throw new WERR_INVALID_OPERATION('Snapshot source schema version is unavailable') + } + return row.name +} + +/** Read source metadata and index ownership in the caller's already pinned view. */ +export async function readKnexSnapshotArchiveHeader(storage: StorageKnex, identityKey: string, k: Knex) { + const generation = await readGenerationIndexState(k, storage.knex.client.config.migrations) + const sourceStorage = await storage.readSettings(k) + const user = await storage.findUserByIdentityKey(identityKey, k) + if (user === undefined) throw new WERR_INVALID_PARAMETER('identityKey', 'an existing wallet profile') + return { + header: { + sourceStorage, + user, + sourceSchema: await readSnapshotArchiveSourceSchema(storage, k) + }, + profileIndexes: generation ?? (await readSnapshotProfileIndexState(k, storage.knex.client.config.migrations)), + relationIndexes: generation ?? (await readSnapshotRelationIndexState(k, storage.knex.client.config.migrations)), + globalIndexes: generation ?? (await readSnapshotGlobalIndexState(k, storage.knex.client.config.migrations)), + certificateIndexes: + generation ?? (await readSnapshotCertificateIndexState(k, storage.knex.client.config.migrations)) + } +} + +export type KnexSnapshotArchiveHeader = Awaited> + +/** Construct a handle only after its caller has proved capture publication. */ +export function createKnexSnapshotArchiveSource( + storage: StorageKnex, + view: RetainedReadSnapshot, + { header, profileIndexes, relationIndexes, certificateIndexes, globalIndexes }: KnexSnapshotArchiveHeader +): SnapshotArchiveSource { + const userId = header.user.userId + const snapshotId = Utils.toHex(Random(32)) + return { + version: 1, + snapshotId, + ...header, + expiresAt: view.expiresAt, + get isOpen() { + return view.isOpen + }, + closed: view.closed, + close: view.close, + readPage: createKnexWalletSnapshotPageReader( + storage, + userId, + snapshotId, + view, + profileIndexes, + relationIndexes, + certificateIndexes, + globalIndexes + ), + validateClosure: async () => { + await view.read(trx => + assertKnexSnapshotArchiveClosure( + storage.toDb(trx), + userId, + profileIndexes, + relationIndexes, + certificateIndexes, + globalIndexes + ) + ) + } + } +} + +/** + * Internal SQL capture source. The caller supplies a dedicated reader provider, + * separate from the staging writer, and awaits close before releasing it. + * No capability is advertised by constructing this local source. + */ +export async function openKnexSnapshotArchiveSource( + storage: StorageKnex, + identityKey: string, + options: WalletReadSnapshotOptions = {}, + openView: () => Promise = () => storage.openReadSnapshot(options) +): Promise { + if (typeof identityKey !== 'string' || !/^(02|03)[0-9a-fA-F]{64}$/.test(identityKey)) { + throw new WERR_INVALID_PARAMETER('identityKey', 'a compressed public identity key') + } + const view = await openView() + try { + const header = await view.read(trx => readKnexSnapshotArchiveHeader(storage, identityKey, storage.toDb(trx))) + return createKnexSnapshotArchiveSource(storage, view, header) + } catch (error) { + await view.close().catch(() => undefined) + throw error + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts new file mode 100644 index 000000000..f51810ac5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.property.test.ts @@ -0,0 +1,248 @@ +import fc from 'fast-check' +import { createHash } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { decodeSyncTransfer } from '../../remoting/SyncTransfer' +import { captureKnexSnapshotArchive } from './captureKnexSnapshotArchive' +import { addSnapshotArchiveTables } from '../../schema/snapshotArchiveMigration' +import { + KnexSnapshotArchiveStore, + snapshotArchiveTables, + type SnapshotArchiveBinding +} from './KnexSnapshotArchiveStore' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +const identity = '02' + '11'.repeat(32) +const other = '03' + '22'.repeat(32) +const date = new Date('2026-01-01T00:00:00.000Z') +const binding: SnapshotArchiveBinding = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceSchema: 'property-v1', + sourceStorage: { + created_at: date, + updated_at: date, + storageIdentityKey: 'original', + storageName: 'source', + chain: 'test', + dbtype: 'SQLite', + maxOutputScript: 1024 + }, + user: { userId: 1, identityKey: identity, activeStorage: 'historical', created_at: date, updated_at: date } +} +const digest = (bytes: Uint8Array | string): string => createHash('sha256').update(bytes).digest('hex') + +test('generated SQL captures match the selected profile or leave no readable state after cancellation', async () => { + const directory = await mkdtemp(join(tmpdir(), 'wallet-capture-property-')) + const open = () => + new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) + const writer = open() + const reader = open() + try { + await writer.knex.raw('PRAGMA journal_mode = WAL') + await writer.migrate('generated capture', 'generated-source') + await writer.makeAvailable() + await reader.makeAvailable() + const { user: first } = await writer.findOrInsertUser(identity) + const { user: second } = await writer.findOrInsertUser(other) + const store = new KnexSnapshotArchiveStore(writer.knex) + await fc.assert( + fc.asyncProperty( + fc.uniqueArray(fc.integer({ min: 0, max: 10000 }), { maxLength: 8 }), + fc.uniqueArray(fc.integer({ min: 0, max: 10000 }), { maxLength: 8 }), + fc.boolean(), + fc.integer({ min: -1, max: 14 }), + async (firstLabels, secondLabels, chooseSecond, cancelAfter) => { + await writer.knex('tx_labels').delete() + const rows = [ + ...firstLabels.map(label => ({ userId: first.userId, label: `first-${label}` })), + ...secondLabels.map(label => ({ userId: second.userId, label: `second-${label}` })) + ].map(row => ({ + ...row, + isDeleted: row.label.endsWith('0'), + created_at: date.toISOString(), + updated_at: date.toISOString() + })) + if (rows.length > 0) await writer.knex('tx_labels').insert(rows) + const selected = chooseSecond ? second : first + const expected = rows + .filter(row => row.userId === selected.userId) + .map(row => ({ label: row.label, isDeleted: row.isDeleted })) + const signal = new AbortController() + if (cancelAfter === 0) signal.abort() + const seen: number[] = [] + const pending = captureKnexSnapshotArchive(reader, writer.knex, selected.identityKey, 'test', { + signal: signal.signal, + onProgress: progress => { + seen.push(progress.pages) + if (progress.pages === cancelAfter) signal.abort() + progress.pages = -100 // A consumer cannot mutate the controller's cursor. + } + }) + if (cancelAfter >= 0 && cancelAfter <= 13) await expect(pending).rejects.toThrow('cancelled') + else { + const manifest = await pending + expect(manifest.pages).toBe(13) + expect(manifest.binding.user.identityKey).toBe(selected.identityKey) + const page = await store.read(selected.identityKey, manifest.archiveId, 8) + const frame = decodeSyncTransfer(page.bytes) as { + table: string + rows: Array<{ label: string; isDeleted: boolean }> + } + expect(frame.table).toBe('txLabels') + expect(frame.rows.map(row => ({ label: row.label, isDeleted: row.isDeleted }))).toEqual(expected) + await expect(store.read(chooseSecond ? identity : other, manifest.archiveId, 8)).rejects.toThrow( + 'unavailable' + ) + await store.close(selected.identityKey, manifest.archiveId) + } + expect(seen).toEqual(Array.from({ length: seen.length }, (_, index) => index + 1)) + expect(await writer.knex('snapshot_archives')).toHaveLength(0) + expect(await writer.knex('snapshot_archive_pages')).toHaveLength(0) + expect(await writer.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 0, + reservedBytes: 0 + }) + } + ), + { seed: Number.isSafeInteger(requestedSeed) ? requestedSeed : 5442026 } + ) + } finally { + await reader.destroy() + await writer.destroy() + await rm(directory, { recursive: true, force: true }) + } +}, 120000) + +// The model uses Node's independent hash implementation and only the persisted +// public receipts; it does not call the store's hashing/accounting helpers. +test('generated capture schedules preserve immutable receipts and reserve capacity through cancellation or expiry', async () => { + const db = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const store = new KnexSnapshotArchiveStore(db) + try { + await addSnapshotArchiveTables(db) + await fc.assert( + fc.asyncProperty( + fc.array( + fc.record({ + bytes: fc.uint8Array({ minLength: 1, maxLength: 96 }), + rows: fc.integer({ min: 0, max: 7 }), + replay: fc.boolean(), + reconnect: fc.boolean() + }), + { minLength: 13, maxLength: 13 } + ), + fc.integer({ min: 4097, max: 16000 }), + fc.integer({ min: -1, max: 13 }), + fc.boolean(), + async (pages, reservation, cancelAfter, expire) => { + let used = new TextEncoder().encode(JSON.stringify(binding)).length + 4096 + if (reservation < used) { + await expect(store.begin(binding, { maxBytes: reservation })).rejects.toThrow('metadata') + expect(await db('snapshot_archives')).toHaveLength(0) + return + } + const writer = await store.begin(binding, { maxBytes: reservation }) + let active = store + let expectedDigest = digest(JSON.stringify(binding)) + let rows = 0 + let accepted = 0 + try { + for (const [sequence, generated] of pages.entries()) { + if (cancelAfter === sequence) return + if (generated.reconnect) active = new KnexSnapshotArchiveStore(db) + const page = { + sequence, + table: snapshotArchiveTables[sequence], + rows: generated.rows, + done: true, + bytes: generated.bytes + } + const charge = generated.bytes.length + 512 + if (used + charge > reservation) { + const before = await db('snapshot_archives').first() + await expect(active.append(writer, page)).rejects.toThrow('reservation exhausted') + expect(await db('snapshot_archives').first()).toEqual(before) + return + } + await active.append(writer, page) + accepted++ + used += charge + rows += generated.rows + expectedDigest = digest( + JSON.stringify([expectedDigest, sequence, page.table, page.rows, true, digest(page.bytes)]) + ) + if (generated.replay) await active.append(writer, page) + const persisted = await db('snapshot_archives').first() + expect(persisted).toMatchObject({ + nextSequence: accepted, + tableIndex: accepted, + usedBytes: used, + rows, + digest: expectedDigest + }) + expect(await db('snapshot_archive_pages')).toHaveLength(accepted) + await expect(active.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') + } + if (cancelAfter === 13) return + const manifest = await active.seal(writer) + expect(manifest).toMatchObject({ pages: 13, rows, digest: expectedDigest, binding }) + await active.close(other, writer.archiveId) + for (const [sequence, generated] of pages.entries()) { + const received = await new KnexSnapshotArchiveStore(db).read(identity, writer.archiveId, sequence) + expect(received).toMatchObject({ + bytes: generated.bytes, + rows: generated.rows, + digest: digest(generated.bytes), + done: true, + table: snapshotArchiveTables[sequence] + }) + } + if (expire) { + await db('snapshot_archives').where({ archiveId: writer.archiveId }).update({ expiresAt: 0 }) + await expect(active.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') + await expect(active.read(identity, writer.archiveId, 0)).rejects.toThrow('unavailable') + } + } finally { + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ + archives: 1, + reservedBytes: reservation + }) + await new KnexSnapshotArchiveStore(db).close(identity, writer.archiveId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + } + } + ) + ) + } finally { + await db.destroy() + } +}, 60000) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.test.ts new file mode 100644 index 000000000..bcae6af7f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.test.ts @@ -0,0 +1,735 @@ +import { WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { KnexMigrations } from '../../schema/KnexMigrations' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { addSnapshotArchiveTables, removeSnapshotArchiveTables } from '../../schema/snapshotArchiveMigration' +import { addSnapshotArchiveOwnerTable } from '../../schema/snapshotArchiveOwnerMigration' +import { lockSnapshotArchiveCapacity } from './SnapshotArchiveSql' +import { + assignSnapshotArchiveOwner, + releaseSnapshotArchiveOwner, + reserveSnapshotArchiveOwner, + SnapshotArchiveCleanupPendingError +} from './SnapshotArchiveOwner' +import { verifySnapshotArchiveDirectory, verifySnapshotArchivePage } from './SnapshotArchiveDirectory' +import { + KnexSnapshotArchiveStore, + snapshotArchiveTables, + snapshotArchiveLimits, + type SnapshotArchiveBinding, + type SnapshotArchiveWriter +} from './KnexSnapshotArchiveStore' + +const identity = '02' + '11'.repeat(32) +const other = '03' + '22'.repeat(32) +const date = new Date('2026-01-01T00:00:00.000Z') +const binding: SnapshotArchiveBinding = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceSchema: 'test-schema-v1', + sourceStorage: { + created_at: date, + updated_at: date, + storageIdentityKey: 'source', + storageName: 'original source', + chain: 'test', + dbtype: 'SQLite', + maxOutputScript: 1024 + }, + user: { userId: 7, identityKey: identity, activeStorage: 'historical primary', created_at: date, updated_at: date } +} +const bytes = new TextEncoder().encode('[]') +const databases: Knex[] = [] +const directories: string[] = [] + +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-archive-')) + directories.push(directory) + const open = () => { + const db = knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'archive.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 5000 + }) + databases.push(db) + return db + } + const db = open() + await db.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(db) + return { db, peer: open(), store: new KnexSnapshotArchiveStore(db) } +} + +async function complete(store: KnexSnapshotArchiveStore, writer: SnapshotArchiveWriter) { + for (const [sequence, table] of snapshotArchiveTables.entries()) { + await store.append(writer, { sequence, table, rows: 0, done: true, bytes }) + } + return await store.seal(writer) +} + +afterEach(async () => { + await Promise.all(databases.splice(0).map(db => db.destroy())) + await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))) +}) + +test('completed staging is immutable and readable from an independent server connection', async () => { + const { db, peer, store } = await fixture() + const writer = await store.begin(binding) + await expect(store.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') + await expect(store.seal(writer)).rejects.toThrow('unavailable') + await expect(new KnexSnapshotArchiveStore(peer).begin(binding)).rejects.toThrow('occupied') + const manifest = await complete(store, writer) + expect(manifest.pages).toBe(13) + expect(manifest.rows).toBe(0) + expect(manifest.binding).toEqual(binding) + expect(manifest).not.toHaveProperty('writerToken') + const secondServer = new KnexSnapshotArchiveStore(peer) + expect(await secondServer.inspect(identity, writer.archiveId)).toEqual(manifest) + expect(await secondServer.seal(writer)).toEqual(manifest) + for (const [sequence, table] of snapshotArchiveTables.entries()) { + const page = await secondServer.read(identity, writer.archiveId, sequence) + expect(page).toMatchObject({ sequence, table, rows: 0, done: true, bytes }) + page.bytes.fill(0) + expect((await store.read(identity, writer.archiveId, sequence)).bytes).toEqual(bytes) + } + await expect(secondServer.read(identity, writer.archiveId, 13)).rejects.toThrow('unavailable') + await expect( + secondServer.append(writer, { sequence: 13, table: 'syncStates', rows: 0, done: true, bytes }) + ).rejects.toThrow('unavailable') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ + archives: 1, + reservedBytes: snapshotArchiveLimits.archiveBytes + }) + await secondServer.close(identity, writer.archiveId) + await secondServer.close(identity, writer.archiveId) + await expect(store.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('mainnet source metadata survives sealing and cross-connection reads', async () => { + const { peer, store } = await fixture() + const mainnet: SnapshotArchiveBinding = { + ...binding, + sourceStorage: { ...binding.sourceStorage, chain: 'main' } + } + const writer = await store.begin(mainnet) + const manifest = await complete(store, writer) + const secondServer = new KnexSnapshotArchiveStore(peer) + expect(manifest.binding).toEqual(mainnet) + expect((await secondServer.inspect(identity, writer.archiveId)).binding).toEqual(mainnet) + expect((await secondServer.read(identity, writer.archiveId, 12)).table).toBe('syncStates') + await secondServer.close(identity, writer.archiveId) + await expect(store.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') +}) + +test('cleanup retains live source pages and quota while reaping an unrelated expired archive', async () => { + const { db, peer, store } = await fixture() + await addSnapshotArchiveOwnerTable(db) + const writer = await store.begin(binding) + await complete(store, writer) + const owner = { identityKey: identity, requestId: '1'.repeat(64), claimToken: '2'.repeat(64) } + await db.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + await reserveSnapshotArchiveOwner(trx, owner) + await assignSnapshotArchiveOwner(trx, owner, writer.archiveId) + }) + const independent = await store.begin({ ...binding, user: { ...binding.user, identityKey: other } }) + await complete(store, independent) + await db('snapshot_archives').update({ expiresAt: 0 }) + await expect(store.close(identity, writer.archiveId)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect(await db('snapshot_archive_pages').where('archiveId', writer.archiveId)).toHaveLength(13) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 2 }) + const replacement = new KnexSnapshotArchiveStore(peer) + await replacement.reap() + expect(await db('snapshot_archives').select('archiveId', 'state')).toEqual([ + { archiveId: writer.archiveId, state: 'closing' } + ]) + expect(await db('snapshot_archive_pages').where('archiveId', writer.archiveId)).toHaveLength(13) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ + archives: 1, + reservedBytes: snapshotArchiveLimits.archiveBytes + }) + await releaseSnapshotArchiveOwner(peer, owner) + await replacement.reap() + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('reaping exposes storage failure and retains capacity until a later successful cleanup', async () => { + const { db, peer, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + await db('snapshot_archives').where('archiveId', writer.archiveId).update({ expiresAt: 0 }) + const failure = new Error('synthetic page deletion failure') + const interrupt = (query: { sql: string }): void => { + if (query.sql.startsWith('delete from `snapshot_archive_pages`')) throw failure + } + peer.on('query', interrupt) + const replacement = new KnexSnapshotArchiveStore(peer) + try { + await expect(replacement.reap()).rejects.toBe(failure) + } finally { + peer.off('query', interrupt) + } + expect(await db('snapshot_archives').first('state')).toEqual({ state: 'closing' }) + expect(await db('snapshot_archive_pages')).toHaveLength(13) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ + archives: 1, + reservedBytes: snapshotArchiveLimits.archiveBytes + }) + await replacement.reap() + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('a replacement server returns verified inclusion metadata without selecting payloads or writer credentials', async () => { + const { db, peer, store } = await fixture() + const writer = await store.begin(binding) + await expect(store.directory(identity, writer.archiveId)).rejects.toThrow('unavailable') + const manifest = await complete(store, writer) + const second = new KnexSnapshotArchiveStore(peer) + const queries: string[] = [] + const observe = (query: { sql: string }) => queries.push(query.sql) + peer.on('query', observe) + const directory = await second.directory(identity, writer.archiveId) + peer.off('query', observe) + const pageQueries = queries.filter(sql => sql.includes('snapshot_archive_pages')) + expect(pageQueries).toHaveLength(1) + expect(pageQueries[0]).not.toMatch(/payload|\*/) + expect(pageQueries[0]).toContain('limit ?') + expect(JSON.stringify(directory)).not.toContain(writer.writerToken) + expect(directory.bindingJson).toBe((await db('snapshot_archives').first()).binding) + const verified = verifySnapshotArchiveDirectory(directory, { + identityKey: identity, + chain: 'test', + sourceStorageIdentityKey: 'source', + archiveId: manifest.archiveId, + digest: manifest.digest + }) + expect(verified.manifest).toEqual(manifest) + expect(verified.tables.syncStates).toEqual({ first: 12, pages: 1, rows: 0 }) + const last = await second.read(identity, writer.archiveId, verified.tables.syncStates.first) + expect(verifySnapshotArchivePage(last, verified.receipts[12])).toEqual(bytes) + directory.receipts[0].digest = 'f'.repeat(64) + expect((await second.directory(identity, writer.archiveId)).receipts[0].digest).not.toBe('f'.repeat(64)) + await expect(second.directory(other, writer.archiveId)).rejects.toThrow('unavailable') + await store.close(identity, writer.archiveId) + await expect(second.directory(identity, writer.archiveId)).rejects.toThrow('unavailable') +}) + +test('a directory read refuses a missing staged receipt', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + await db('snapshot_archive_pages').where({ archiveId: writer.archiveId, sequence: 7 }).delete() + await expect(store.directory(identity, writer.archiveId)).rejects.toThrow('unavailable') +}) + +test.each(['closing', 'expired'])('a directory read rechecks %s after loading receipts', async state => { + const { db, peer, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + const original = db.client.query.bind(db.client) + const intercept = jest.spyOn(db.client, 'query').mockImplementation(async (connection, query, ...rest) => { + const result = await original(connection, query, ...rest) + if (typeof query !== 'string' && query.sql.includes('from `snapshot_archive_pages`')) { + await peer('snapshot_archives') + .where({ archiveId: writer.archiveId }) + .update(state === 'closing' ? { state: 'closing' } : { expiresAt: 0 }) + } + return result + }) + try { + await expect(store.directory(identity, writer.archiveId)).rejects.toThrow('unavailable') + } finally { + intercept.mockRestore() + } +}) + +test('profile and internal capture ownership remain independent authorization boundaries', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await expect( + store.append( + { ...writer, writerToken: '0'.repeat(64) }, + { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes } + ) + ).rejects.toThrow('unavailable') + await complete(store, writer) + await expect(store.inspect(other, writer.archiveId)).rejects.toThrow('unavailable') + await expect(store.read(other, writer.archiveId, 0)).rejects.toThrow('unavailable') + await store.close(other, writer.archiveId) + expect((await store.inspect(identity, writer.archiveId)).pages).toBe(13) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1 }) +}) + +test('lost page acknowledgements replay exactly, rejecting gaps and changed content or metadata', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + const first = { sequence: 0, table: 'provenTxs' as const, rows: 1, done: false, bytes: new Uint8Array([1, 2, 3]) } + await store.append(writer, first) + const before = await db('snapshot_archives').first() + await store.append(writer, first) + expect(await db('snapshot_archives').first()).toEqual(before) + expect(await db('snapshot_archive_pages')).toHaveLength(1) + for (const change of [{ bytes }, { rows: 2 }, { done: true }, { table: 'outputs' as const }, { sequence: 2 }]) { + await expect(store.append(writer, { ...first, ...change })).rejects.toThrow('unavailable') + } + expect(await db('snapshot_archives').first()).toEqual(before) + await expect(store.append(writer, { ...first, sequence: 1, table: 'outputs' })).rejects.toThrow('unavailable') + await store.append(writer, { ...first, sequence: 1, rows: 0, done: true }) + expect((await db('snapshot_archives').first()).tableIndex).toBe(1) +}) + +test('an interrupted atomic append preserves the prior cursor, accounting and pages', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + const before = await db('snapshot_archives').first() + await db.raw( + "CREATE TRIGGER archive_checkpoint_failure BEFORE UPDATE ON snapshot_archives WHEN NEW.nextSequence > OLD.nextSequence BEGIN SELECT RAISE(ABORT, 'synthetic checkpoint failure'); END" + ) + await expect(store.append(writer, { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes })).rejects.toThrow( + 'synthetic checkpoint failure' + ) + await db.raw('DROP TRIGGER archive_checkpoint_failure') + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archives').first()).toEqual(before) + await store.append(writer, { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes }) + expect(await db('snapshot_archive_pages')).toHaveLength(1) +}) + +test('quota exhaustion rejects before persistence and retains the original reservation', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding, { maxBytes: 8192 }) + const before = await db('snapshot_archives').first() + await expect( + store.append(writer, { sequence: 0, table: 'provenTxs', rows: 1, done: true, bytes: new Uint8Array(8192) }) + ).rejects.toThrow('reservation exhausted') + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archives').first()).toEqual(before) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 8192 }) + await store.close(identity, writer.archiveId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('independent profiles enforce both global byte and handle limits', async () => { + const { db, peer, store } = await fixture() + const owners: Array<{ identityKey: string; writer: SnapshotArchiveWriter }> = [] + for (let n = 0; n < 4; n++) { + const identityKey = '02' + n.toString(16).padStart(64, '0') + owners.push({ identityKey, writer: await store.begin({ ...binding, user: { ...binding.user, identityKey } }) }) + } + await expect(new KnexSnapshotArchiveStore(peer).begin(binding)).rejects.toThrow('occupied') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 4, reservedBytes: 128 * 1024 * 1024 }) + for (const owner of owners) await store.close(owner.identityKey, owner.writer.archiveId) + for (let n = 0; n < 8; n++) { + await store.begin( + { ...binding, user: { ...binding.user, identityKey: '02' + n.toString(16).padStart(64, '0') } }, + { maxBytes: 8192 } + ) + } + await expect(new KnexSnapshotArchiveStore(peer).begin(binding, { maxBytes: 8192 })).rejects.toThrow('occupied') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 8, reservedBytes: 65536 }) +}) + +test('expiry from an independent store never publishes partial staging and reclaims its reservation', async () => { + const { db, peer, store } = await fixture() + const writer = await store.begin(binding) + await store.append(writer, { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes }) + const restarted = new KnexSnapshotArchiveStore(peer) + await expect(restarted.inspect(identity, writer.archiveId)).rejects.toThrow('unavailable') + await db('snapshot_archives').where({ archiveId: writer.archiveId }).update({ expiresAt: 0 }) + await expect( + store.append(writer, { sequence: 1, table: 'provenTxReqs', rows: 0, done: true, bytes }) + ).rejects.toThrow('unavailable') + await expect(restarted.seal(writer)).rejects.toThrow('unavailable') + await restarted.reap() + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('interrupted bounded cleanup remains recoverable and does not release capacity early', async () => { + const { db, peer, store } = await fixture() + const writer = await store.begin(binding) + for (let sequence = 0; sequence < 70; sequence++) { + await store.append(writer, { sequence, table: 'provenTxs', rows: 1, done: false, bytes }) + } + const original = db.client.query.bind(db.client) + let deletions = 0 + const failure = new Error('synthetic deletion failure') + const intercept = jest.spyOn(db.client, 'query').mockImplementation(async (connection, query, ...rest) => { + if (typeof query !== 'string' && query.sql.startsWith('delete from `snapshot_archive_pages`')) { + if (++deletions === 2) throw failure + expect(query.bindings.length).toBeLessThanOrEqual(33) + } + return await original(connection, query, ...rest) + }) + await expect(store.close(identity, writer.archiveId)).rejects.toBe(failure) + intercept.mockRestore() + expect(await db('snapshot_archive_pages')).toHaveLength(38) + expect((await db('snapshot_archives').first()).state).toBe('closing') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ + archives: 1, + reservedBytes: snapshotArchiveLimits.archiveBytes + }) + const restarted = new KnexSnapshotArchiveStore(peer) + await expect(restarted.begin(binding)).rejects.toThrow('occupied') + await restarted.reap() + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + await restarted.reap() + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('corrupted staged bytes are never returned as a verified page', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + await db('snapshot_archive_pages') + .where({ archiveId: writer.archiveId, sequence: 0 }) + .update({ payload: Buffer.from([0]) }) + await expect(store.read(identity, writer.archiveId, 0)).rejects.toThrow('unavailable') +}) + +test('staging migration recovers partial auxiliary DDL without resetting existing reservations', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await addSnapshotArchiveTables(db) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1 }) + expect((await db('snapshot_archives').first()).archiveId).toBe(writer.archiveId) + await store.close(identity, writer.archiveId) + await db.schema.dropTable('snapshot_archive_pages') + await addSnapshotArchiveTables(db) + expect(await db.schema.hasTable('snapshot_archive_pages')).toBe(true) + await removeSnapshotArchiveTables(db) + for (const table of ['snapshot_archive_pages', 'snapshot_archives', 'snapshot_archive_capacity']) { + expect(await db.schema.hasTable(table)).toBe(false) + } +}) + +test('the exact page and reservation limits admit the boundary and reject the next byte', async () => { + const { db, store } = await fixture() + const payload = new Uint8Array(1024 * 1024).fill(7) + const exactBytes = new TextEncoder().encode(JSON.stringify(binding)).length + 4096 + 512 + payload.length + const writer = await store.begin(binding, { maxBytes: exactBytes }) + await store.append(writer, { sequence: 0, table: 'provenTxs', rows: 1000, done: false, bytes: payload }) + expect(Number((await db('snapshot_archives').first()).usedBytes)).toBe(exactBytes) + await expect(store.append(writer, { sequence: 1, table: 'provenTxs', rows: 1, done: false, bytes })).rejects.toThrow( + 'reservation exhausted' + ) + await store.close(identity, writer.archiveId) + const next = await store.begin(binding) + await expect( + store.append(next, { + sequence: 0, + table: 'provenTxs', + rows: 1, + done: true, + bytes: new Uint8Array(payload.length + 1) + }) + ).rejects.toThrow('page') + expect(await db('snapshot_archive_pages')).toHaveLength(0) +}) + +test('caller mutation cannot rebind a pending archive or replace pending page bytes', async () => { + const { db, store } = await fixture() + const mutable = structuredClone(binding) + const opening = store.begin(mutable) + mutable.user.identityKey = other + mutable.user.activeStorage = 'changed' + mutable.sourceStorage.storageIdentityKey = 'changed' + const writer = await opening + expect((await db('snapshot_archives').first()).identityKey).toBe(identity) + const payload = new Uint8Array([1, 2, 3]) + const append = store.append(writer, { sequence: 0, table: 'provenTxs', rows: 1, done: false, bytes: payload }) + payload.fill(0) + await append + expect(new Uint8Array((await db('snapshot_archive_pages').first()).payload)).toEqual(new Uint8Array([1, 2, 3])) + expect(JSON.parse((await db('snapshot_archives').first()).binding).user.activeStorage).toBe('historical primary') +}) + +test.each([0, -1, 0.5, Number.NaN, Number.POSITIVE_INFINITY, 3600001])( + 'invalid lifetime %s never reserves capacity', + async lifetimeMs => { + const { db, store } = await fixture() + await expect(store.begin(binding, { lifetimeMs })).rejects.toThrow('lifetimeMs') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + } +) + +test.each([0, 4096, 1.5, Number.NaN, 33554433])( + 'invalid byte reservation %s never persists a handle', + async maxBytes => { + const { db, store } = await fixture() + await expect(store.begin(binding, { maxBytes })).rejects.toThrow('maxBytes') + expect(await db('snapshot_archives')).toHaveLength(0) + } +) + +test('metadata limits and original schema/profile bindings reject malformed inputs', async () => { + const { db, store } = await fixture() + const variants = [ + { ...binding, version: 2 }, + { ...binding, snapshotId: '' }, + { ...binding, sourceSchema: '' }, + { ...binding, sourceSchema: 'x'.repeat(257) }, + { ...binding, sourceSchema: 1 }, + { ...binding, sourceStorage: { ...binding.sourceStorage, chain: 'other' } }, + { ...binding, sourceStorage: { ...binding.sourceStorage, storageIdentityKey: '' } }, + { ...binding, sourceStorage: { ...binding.sourceStorage, storageIdentityKey: 'x'.repeat(131) } }, + { ...binding, sourceStorage: { ...binding.sourceStorage, storageIdentityKey: 1 } }, + { ...binding, sourceStorage: { ...binding.sourceStorage, created_at: new Date(Number.NaN) } }, + { ...binding, user: { ...binding.user, updated_at: '2026-01-01' } }, + { ...binding, user: { ...binding.user, userId: 0 } }, + { ...binding, user: { ...binding.user, identityKey: 'invalid' } } + ] + for (const value of variants) + await expect(store.begin(value as SnapshotArchiveBinding)).rejects.toBeInstanceOf(WERR_INVALID_PARAMETER) + await expect( + store.begin({ ...binding, sourceStorage: { ...binding.sourceStorage, storageName: 'x'.repeat(65536) } }) + ).rejects.toThrow('metadata') + await expect(store.begin(binding, { maxBytes: 4097 })).rejects.toThrow('metadata') + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('invalid page shapes do not advance the capture', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + const first = { sequence: 0, table: 'provenTxs' as const, rows: 1, done: false, bytes } + const variants = [ + { sequence: -1 }, + { sequence: 4096 }, + { rows: -1 }, + { rows: 1001 }, + { rows: 0 }, + { done: undefined }, + { bytes: [] }, + { bytes: new Uint8Array() }, + { table: 'users' } + ] + for (const variant of variants) + await expect(store.append(writer, { ...first, ...variant } as typeof first)).rejects.toBeInstanceOf( + WERR_INVALID_PARAMETER + ) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + expect((await db('snapshot_archives').first()).nextSequence).toBe(0) + for (const bad of ['', 'a'.repeat(63), 'g'.repeat(64)]) { + await expect(store.inspect(identity, bad)).rejects.toThrow('archiveId') + await expect(store.close(identity, bad)).rejects.toThrow('archiveId') + await expect(store.seal({ ...writer, writerToken: bad })).rejects.toThrow('writerToken') + } + await expect(store.inspect('invalid', writer.archiveId)).rejects.toThrow('identityKey') +}) + +test('missing capacity metadata fails closed before creating a capture', async () => { + const { db, store } = await fixture() + await db('snapshot_archive_capacity').delete() + await expect(store.begin(binding)).rejects.toThrow('schema is unavailable') + expect(await db('snapshot_archives')).toHaveLength(0) +}) + +test('an expiry or close between header and payload reads cannot return a newly stale page', async () => { + const { db, peer, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + const original = db.client.query.bind(db.client) + let intercepted = false + const intercept = jest.spyOn(db.client, 'query').mockImplementation(async (connection, query, ...rest) => { + if (!intercepted && typeof query !== 'string' && query.sql.startsWith('select * from `snapshot_archive_pages`')) { + intercepted = true + await peer('snapshot_archives').where({ archiveId: writer.archiveId }).update({ expiresAt: 0 }) + } + return await original(connection, query, ...rest) + }) + await expect(store.read(identity, writer.archiveId, 0)).rejects.toThrow('unavailable') + intercept.mockRestore() + expect(intercepted).toBe(true) +}) + +test('missing persisted page content cannot become an acknowledged replay or read', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + await db('snapshot_archive_pages').where({ archiveId: writer.archiveId, sequence: 0 }).delete() + await expect(store.append(writer, { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes })).rejects.toThrow( + 'unavailable' + ) + await expect(store.read(identity, writer.archiveId, 0)).rejects.toThrow('unavailable') +}) + +test('capture cannot append or seal once cleanup has begun', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + await db('snapshot_archives').where({ archiveId: writer.archiveId }).update({ state: 'closing' }) + await expect(store.append(writer, { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes })).rejects.toThrow( + 'unavailable' + ) + await expect(store.seal(writer)).rejects.toThrow('unavailable') + await store.reap() + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('the MySQL adapter uses the shared database clock and a blob type admitting bounded pages', async () => { + const { db, store } = await fixture() + const raw = jest.fn(async () => [[{ now: '1790770000000' }]]) + const readClock = Reflect.get(store, 'now') as (database: unknown) => Promise + expect(await readClock.call(store, { client: { config: { client: 'mysql2' } }, raw })).toBe(1790770000000) + expect(raw).toHaveBeenCalledWith('SELECT FLOOR(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000) AS now') + await db.schema.dropTable('snapshot_archive_pages') + const original = db.client.config.client + try { + // Keep the SQLite executor while checking the dialect-dependent declared + // type. Native MySQL qualification independently executes the emitted DDL. + db.client.config.client = 'mysql2' + await addSnapshotArchiveTables(db) + } finally { + db.client.config.client = original + } + const columns: Array<{ name: string; type: string }> = await db.raw('PRAGMA table_info(snapshot_archive_pages)') + expect(columns.find(column => column.name === 'payload')?.type.toUpperCase()).toBe('MEDIUMBLOB') +}) + +test('parallel profile captures never overwrite, seal, or collect each other', async () => { + const { db, store } = await fixture() + const second = await store.begin({ ...binding, user: { ...binding.user, identityKey: other } }, { maxBytes: 16384 }) + const secondBefore = await db('snapshot_archives').where({ archiveId: second.archiveId }).first() + const first = await store.begin(binding, { maxBytes: 32768 }) + await complete(store, first) + expect(await db('snapshot_archives').where({ archiveId: second.archiveId }).first()).toEqual(secondBefore) + const secondManifest = await complete(store, second) + const secondPages = await db('snapshot_archive_pages').where({ archiveId: second.archiveId }).orderBy('sequence') + const projections: string[] = [] + const observe = (query: { sql: string }): void => { + if (query.sql.startsWith('select') && query.sql.includes('snapshot_archive_pages')) projections.push(query.sql) + } + db.on('query', observe) + await Promise.all([store.close(identity, first.archiveId), store.close(identity, first.archiveId)]) + db.removeListener('query', observe) + expect(projections.length).toBeGreaterThan(0) + expect(projections.every(query => query.startsWith('select `sequence` from'))).toBe(true) + expect(await store.inspect(other, second.archiveId)).toEqual(secondManifest) + expect(await db('snapshot_archive_pages').where({ archiveId: second.archiveId }).orderBy('sequence')).toEqual( + secondPages + ) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 16384 }) + await store.close(other, second.archiveId) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) +}) + +test('valid identifier and metadata boundary values remain admitted', async () => { + const { store } = await fixture() + for (const sourceSchema of ['x', 'x'.repeat(256)]) { + for (const storageIdentityKey of ['x', 'x'.repeat(130)]) { + const writer = await store.begin({ + ...binding, + sourceSchema, + sourceStorage: { ...binding.sourceStorage, storageIdentityKey } + }) + await store.close(identity, writer.archiveId) + } + } + const base = new TextEncoder().encode(JSON.stringify(binding)).length + const large = { + ...binding, + sourceStorage: { + ...binding.sourceStorage, + storageName: 'x'.repeat(65536 - base + binding.sourceStorage.storageName.length) + } + } + expect(new TextEncoder().encode(JSON.stringify(large))).toHaveLength(65536) + const writer = await store.begin(large, { maxBytes: 65536 + 4096 }) + await store.close(identity, writer.archiveId) +}) + +test('identifiers reject prefixes, suffixes and non-string coercions before SQL', async () => { + const { store } = await fixture() + const writer = await store.begin(binding) + for (const bad of ['x' + writer.archiveId, writer.archiveId + 'x', { toString: () => writer.archiveId }]) { + await expect(store.inspect(identity, bad as string)).rejects.toBeInstanceOf(WERR_INVALID_PARAMETER) + await expect(store.seal({ ...writer, archiveId: bad as string })).rejects.toBeInstanceOf(WERR_INVALID_PARAMETER) + } + for (const bad of ['x' + identity, identity + 'x', { toString: () => identity }]) { + await expect(store.inspect(bad as string, writer.archiveId)).rejects.toBeInstanceOf(WERR_INVALID_PARAMETER) + } + for (const malformed of [undefined, { ...binding, user: undefined }, { ...binding, sourceStorage: undefined }]) { + await expect(store.begin(malformed as unknown as SnapshotArchiveBinding)).rejects.toBeInstanceOf( + WERR_INVALID_PARAMETER + ) + } +}) + +test('expiry is inclusive at the shared-clock boundary for both writer and reader', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + const row = await db('snapshot_archives').first() + const readClock = jest + .spyOn(store as unknown as { now: () => Promise }, 'now') + .mockResolvedValue(Number(row.expiresAt)) + try { + await expect(store.seal(writer)).rejects.toThrow('unavailable') + await expect(store.read(identity, writer.archiveId, 0)).rejects.toThrow('unavailable') + } finally { + readClock.mockRestore() + } +}) + +test('only acknowledged sequence positions are readable, even if an unacknowledged page exists', async () => { + const { db, store } = await fixture() + const writer = await store.begin(binding) + await complete(store, writer) + const page = await db('snapshot_archive_pages').where({ archiveId: writer.archiveId, sequence: 0 }).first() + await db('snapshot_archive_pages').insert({ ...page, sequence: 13 }) + await expect(store.read(identity, writer.archiveId, 13)).rejects.toThrow('unavailable') + for (const sequence of [-1, 4096]) { + await expect(store.read(identity, writer.archiveId, sequence)).rejects.toBeInstanceOf(WERR_INVALID_PARAMETER) + await expect( + store.append(writer, { sequence, table: 'provenTxs', rows: 0, done: true, bytes }) + ).rejects.toBeInstanceOf(WERR_INVALID_PARAMETER) + } +}) + +test('the auxiliary migration is registered after the durable sync schema', async () => { + const migrations = new KnexMigrations('test', 'source', 'source', 1024) + expect(await migrations.getLatestMigration()).toBe('2026-10-02-001 repair snapshot SQLite conflict maintenance') +}) + +test('MySQL DDL accommodates the declared metadata and page byte ceilings', async () => { + const mysql = knex({ client: 'mysql2' }) + const commands: string[] = [] + const fake = Object.assign(() => ({ insert: () => ({ onConflict: () => ({ ignore: async () => undefined }) }) }), { + client: mysql.client, + schema: { + hasTable: async () => false, + createTable: async (name: string, define: (table: Knex.CreateTableBuilder) => void) => { + commands.push( + ...mysql.schema + .createTable(name, define) + .toSQL() + .map(query => query.sql) + ) + } + } + }) + try { + await addSnapshotArchiveTables(fake as unknown as Knex) + expect(commands.find(sql => sql.startsWith('create table `snapshot_archives`'))).toContain( + '`binding` mediumtext not null' + ) + expect(commands.find(sql => sql.startsWith('create table `snapshot_archive_pages`'))).toContain( + '`payload` MEDIUMBLOB not null' + ) + } finally { + await mysql.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.ts new file mode 100644 index 000000000..7813bbdef --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/KnexSnapshotArchiveStore.ts @@ -0,0 +1,415 @@ +import { hasSnapshotArchiveOwner, SnapshotArchiveCleanupPendingError } from './SnapshotArchiveOwner' +import { Hash, Random, Utils } from '@bsv/sdk' +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import type { WalletSnapshotTable } from '../WalletReadSnapshot' +import { runInSeries } from '../../../utility/runInSeries' +import { lockSnapshotArchiveCapacity, snapshotArchiveDatabaseNow } from './SnapshotArchiveSql' +import { snapshotArchiveEncoding, type SnapshotArchiveDirectory } from './SnapshotArchiveDirectory' + +import { + snapshotArchiveTables, + snapshotArchiveLimits, + type SnapshotArchiveBinding, + type SnapshotArchiveWriter, + type SnapshotArchiveManifest, + type SnapshotArchivePage +} from './SnapshotArchive' +export { + snapshotArchiveTables, + snapshotArchiveLimits, + type SnapshotArchiveBinding, + type SnapshotArchiveWriter, + type SnapshotArchiveManifest, + type SnapshotArchivePage +} from './SnapshotArchive' + +interface ArchiveRow { + archiveId: string + identityKey: string + writerToken: string + state: 'building' | 'ready' | 'closing' + binding: string + expiresAt: number | string + reservedBytes: number | string + usedBytes: number | string + nextSequence: number + tableIndex: number + rows: number | string + digest: string +} + +interface PageRow { + archiveId: string + sequence: number + tableName: WalletSnapshotTable + rows: number + done: number | boolean + payload: Uint8Array + digest: string +} + +function integer(value: number, min: number, max: number, name: string): number { + if (!Number.isSafeInteger(value) || value < min || value > max) { + throw new WERR_INVALID_PARAMETER(name, `an integer from ${min} to ${max}`) + } + return value +} + +function identifier(value: string, name: string): void { + if (typeof value !== 'string' || !/^[0-9a-f]{64}$/.test(value)) { + throw new WERR_INVALID_PARAMETER(name, 'a version-one snapshot archive identifier') + } +} + +function identity(value: string): void { + if (typeof value !== 'string' || !/^(02|03)[0-9a-fA-F]{64}$/.test(value)) { + throw new WERR_INVALID_PARAMETER('identityKey', 'a compressed public identity key') + } +} + +function validDate(value: Date): boolean { + try { + return Number.isFinite(Date.prototype.getTime.call(value)) + } catch { + return false + } +} + +function hash(bytes: Uint8Array): string { + return Utils.toHex(Hash.sha256(Array.from(bytes))) +} + +function unavailable(): WERR_INVALID_OPERATION { + return new WERR_INVALID_OPERATION('Snapshot archive is unavailable') +} + +/** + * Shared SQL staging with a fixed global reservation and at most one archive per + * profile. Only auxiliary rows take the capacity lock; no source wallet row is + * acquired under it. Capacity remains reserved until every page is deleted. + */ +export class KnexSnapshotArchiveStore { + constructor(private readonly knex: Knex) {} + + private async now(k: Knex): Promise { + return await snapshotArchiveDatabaseNow(k) + } + + private async capacity(k: Knex): Promise<{ archives: number; reservedBytes: number | string }> { + return await lockSnapshotArchiveCapacity(k) + } + + async begin( + binding: SnapshotArchiveBinding, + options: { maxBytes?: number; lifetimeMs?: number } = {} + ): Promise { + const reservedBytes = integer( + options.maxBytes ?? snapshotArchiveLimits.archiveBytes, + snapshotArchiveLimits.headerCharge + 1, + snapshotArchiveLimits.archiveBytes, + 'maxBytes' + ) + const lifetimeMs = integer(options.lifetimeMs ?? 300000, 1, snapshotArchiveLimits.lifetimeMs, 'lifetimeMs') + identity(binding?.user?.identityKey) + identifier(binding?.snapshotId, 'snapshotId') + if ( + binding.version !== 1 || + !['main', 'test'].includes(binding.sourceStorage?.chain) || + typeof binding.sourceSchema !== 'string' || + binding.sourceSchema.length < 1 || + binding.sourceSchema.length > 256 || + typeof binding.sourceStorage.storageIdentityKey !== 'string' || + binding.sourceStorage.storageIdentityKey.length < 1 || + binding.sourceStorage.storageIdentityKey.length > 130 + ) { + throw new WERR_INVALID_PARAMETER('binding', 'a version-one source/profile binding') + } + integer(binding.user.userId, 1, Number.MAX_SAFE_INTEGER, 'userId') + for (const row of [binding.sourceStorage, binding.user]) { + for (const value of [row.created_at, row.updated_at]) { + if (!validDate(value)) { + throw new WERR_INVALID_PARAMETER('binding', 'valid source and profile dates') + } + } + } + const identityKey = binding.user.identityKey + const encoded = JSON.stringify(binding) + const bytes = new TextEncoder().encode(encoded) + const usedBytes = bytes.length + snapshotArchiveLimits.headerCharge + if (bytes.length > snapshotArchiveLimits.bindingBytes || usedBytes > reservedBytes) { + throw new SnapshotResourceLimitError('Snapshot archive metadata exceeds its reservation') + } + const writer = { archiveId: Utils.toHex(Random(32)), writerToken: Utils.toHex(Random(32)) } + await this.knex.transaction(async trx => { + const capacity = await this.capacity(trx) + const occupied = await trx('snapshot_archives').where({ identityKey }).first('archiveId') + if ( + occupied !== undefined || + capacity.archives >= snapshotArchiveLimits.archives || + Number(capacity.reservedBytes) + reservedBytes > snapshotArchiveLimits.totalBytes + ) { + throw new SnapshotResourceLimitError('Snapshot archive capacity is occupied') + } + await trx('snapshot_archives').insert({ + ...writer, + identityKey, + state: 'building', + binding: encoded, + expiresAt: (await this.now(trx)) + lifetimeMs, + reservedBytes, + usedBytes, + nextSequence: 0, + tableIndex: 0, + rows: 0, + digest: hash(bytes) + }) + await trx('snapshot_archive_capacity') + .where({ id: 1 }) + .update({ + archives: capacity.archives + 1, + reservedBytes: Number(capacity.reservedBytes) + reservedBytes + }) + }) + return writer + } + + private async ownedWriter(k: Knex, writer: SnapshotArchiveWriter): Promise { + identifier(writer.archiveId, 'archiveId') + identifier(writer.writerToken, 'writerToken') + const row: ArchiveRow | undefined = await k('snapshot_archives').where(writer).first() + if (row === undefined || row.state === 'closing' || Number(row.expiresAt) <= (await this.now(k))) + throw unavailable() + return row + } + + async append( + writer: SnapshotArchiveWriter, + page: Omit, + authorize?: (trx: Knex) => Promise + ): Promise { + integer(page.sequence, 0, snapshotArchiveLimits.pages - 1, 'sequence') + integer(page.rows, 0, snapshotArchiveLimits.rowsPerPage, 'rows') + if ( + !snapshotArchiveTables.includes(page.table) || + typeof page.done !== 'boolean' || + !(page.bytes instanceof Uint8Array) || + page.bytes.length < 1 || + page.bytes.length > snapshotArchiveLimits.pageBytes || + (page.rows === 0 && !page.done) + ) + throw new WERR_INVALID_PARAMETER('page', 'a bounded snapshot archive page') + // Detach before taking an asynchronous lock, including caller-owned typed bytes. + const owner = { archiveId: writer.archiveId, writerToken: writer.writerToken } + const input = { ...page, bytes: new Uint8Array(page.bytes) } + const digest = hash(input.bytes) + await this.knex.transaction(async trx => { + await this.capacity(trx) + await authorize?.(trx) + const row = await this.ownedWriter(trx, owner) + if (input.sequence < row.nextSequence) { + const prior: PageRow | undefined = await trx('snapshot_archive_pages') + .where({ archiveId: row.archiveId, sequence: input.sequence }) + .first() + if ( + prior?.digest !== digest || + prior.tableName !== input.table || + prior.rows !== input.rows || + Boolean(prior.done) !== input.done + ) + throw unavailable() + return + } + if ( + row.state !== 'building' || + input.sequence !== row.nextSequence || + input.table !== snapshotArchiveTables[row.tableIndex] + ) + throw unavailable() + const usedBytes = Number(row.usedBytes) + input.bytes.length + snapshotArchiveLimits.pageCharge + if (usedBytes > Number(row.reservedBytes)) + throw new SnapshotResourceLimitError('Snapshot archive reservation exhausted') + await trx('snapshot_archive_pages').insert({ + archiveId: row.archiveId, + sequence: input.sequence, + tableName: input.table, + rows: input.rows, + done: input.done, + digest, + payload: Buffer.from(input.bytes) + }) + const receipt = new TextEncoder().encode( + JSON.stringify([row.digest, input.sequence, input.table, input.rows, input.done, digest]) + ) + await trx('snapshot_archives') + .where({ archiveId: row.archiveId }) + .update({ + usedBytes, + nextSequence: row.nextSequence + 1, + tableIndex: row.tableIndex + Number(input.done), + rows: Number(row.rows) + input.rows, + digest: hash(receipt) + }) + }) + } + + private manifest(row: ArchiveRow): SnapshotArchiveManifest { + const binding = JSON.parse(row.binding) as SnapshotArchiveBinding + for (const metadata of [binding.sourceStorage, binding.user]) { + metadata.created_at = new Date(String(metadata.created_at)) + metadata.updated_at = new Date(String(metadata.updated_at)) + } + return { + version: 1, + archiveId: row.archiveId, + binding, + expiresAt: Number(row.expiresAt), + pages: row.nextSequence, + rows: Number(row.rows), + digest: row.digest + } + } + + /** The capture controller must validate the complete source closure before sealing. */ + async seal(writer: SnapshotArchiveWriter): Promise { + const owner = { archiveId: writer.archiveId, writerToken: writer.writerToken } + return await this.knex.transaction(async trx => { + await this.capacity(trx) + const row = await this.ownedWriter(trx, owner) + if (row.tableIndex !== snapshotArchiveTables.length) throw unavailable() + await trx('snapshot_archives').where({ archiveId: row.archiveId }).update({ state: 'ready' }) + return this.manifest(row) + }) + } + + private async ready(identityKey: string, archiveId: string): Promise { + identity(identityKey) + identifier(archiveId, 'archiveId') + const row: ArchiveRow | undefined = await this.knex('snapshot_archives') + .where({ archiveId, identityKey, state: 'ready' }) + .first() + if (row === undefined || Number(row.expiresAt) <= (await this.now(this.knex))) throw unavailable() + return row + } + + async inspect(identityKey: string, archiveId: string): Promise { + return this.manifest(await this.ready(identityKey, archiveId)) + } + + /** Bounded inclusion metadata, without fetching any staged row payloads. */ + async directory(identityKey: string, archiveId: string): Promise { + const header = await this.ready(identityKey, archiveId) + const pages: Array> = await this.knex('snapshot_archive_pages') + .select('sequence', 'tableName', 'rows', 'done', 'digest') + .where({ archiveId }) + .orderBy('sequence') + .limit(snapshotArchiveLimits.pages) + if (pages.length !== header.nextSequence) throw unavailable() + // A concurrent close or expiry must not publish a newly stale directory. + await this.ready(identityKey, archiveId) + return { + version: 1, + encoding: snapshotArchiveEncoding, + archiveId, + expiresAt: Number(header.expiresAt), + pages: header.nextSequence, + rows: Number(header.rows), + digest: header.digest, + bindingJson: header.binding, + receipts: pages.map(page => ({ + sequence: page.sequence, + table: page.tableName, + rows: page.rows, + done: Boolean(page.done), + digest: page.digest + })) + } + } + + async read(identityKey: string, archiveId: string, sequence: number): Promise { + integer(sequence, 0, snapshotArchiveLimits.pages - 1, 'sequence') + const header = await this.ready(identityKey, archiveId) + if (sequence >= header.nextSequence) throw unavailable() + const page: PageRow | undefined = await this.knex('snapshot_archive_pages').where({ archiveId, sequence }).first() + if (page === undefined) throw unavailable() + if (hash(page.payload) !== page.digest) throw unavailable() + // Release/expiry during I/O cannot return a newly stale page. + await this.ready(identityKey, archiveId) + return { + sequence, + table: page.tableName, + rows: page.rows, + done: Boolean(page.done), + digest: page.digest, + bytes: new Uint8Array(page.payload) + } + } + + async close(identityKey: string, archiveId: string): Promise { + identity(identityKey) + identifier(archiveId, 'archiveId') + const found = await this.knex.transaction(async trx => { + await this.capacity(trx) + const row = await trx('snapshot_archives').where({ archiveId, identityKey }).first('archiveId') + if (row === undefined) return 'absent' + await trx('snapshot_archives').where({ archiveId, identityKey }).update({ state: 'closing' }) + return (await hasSnapshotArchiveOwner(trx, { identityKey, archiveId })) ? 'pending' : 'ready' + }) + if (found === 'absent') return + if (found === 'pending') throw new SnapshotArchiveCleanupPendingError() + // Bounded exact-key deletes do not hold source wallet locks or release the + // capacity reservation early. A crash or concurrent closer can resume them. + let hasPages = true + function* pendingBatches(): Generator { + while (hasPages) yield undefined + } + await runInSeries(pendingBatches(), async () => { + const rows: Array<{ sequence: number }> = await this.knex('snapshot_archive_pages') + .select('sequence') + .where({ archiveId }) + .orderBy('sequence') + .limit(32) + hasPages = rows.length > 0 + if (hasPages) + await this.knex('snapshot_archive_pages') + .where({ archiveId }) + .whereIn( + 'sequence', + rows.map(row => row.sequence) + ) + .delete() + }) + await this.knex.transaction(async trx => { + const capacity = await this.capacity(trx) + const row: ArchiveRow | undefined = await trx('snapshot_archives') + .where({ archiveId, identityKey, state: 'closing' }) + .first() + if (row === undefined) return + await trx('snapshot_archives').where({ archiveId }).delete() + await trx('snapshot_archive_capacity') + .where({ id: 1 }) + .update({ + archives: capacity.archives - 1, + reservedBytes: Number(capacity.reservedBytes) - Number(row.reservedBytes) + }) + }) + } + + /** Recover expired or interrupted cleanup without making incomplete captures readable. */ + async reap(): Promise { + const rows: Array<{ archiveId: string; identityKey: string }> = await this.knex('snapshot_archives') + .select('archiveId', 'identityKey') + .where('expiresAt', '<=', await this.now(this.knex)) + .orWhere({ state: 'closing' }) + .orderBy('archiveId') + .limit(snapshotArchiveLimits.archives) + await runInSeries(rows, async row => { + try { + await this.close(row.identityKey, row.archiveId) + } catch (error) { + if (!(error instanceof SnapshotArchiveCleanupPendingError)) throw error + } + }) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotCleanup.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotCleanup.test.ts new file mode 100644 index 000000000..36f7cf588 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotCleanup.test.ts @@ -0,0 +1,205 @@ +import { + drainSnapshotArchiveRequest, + SnapshotArchiveCleanupPendingError, + snapshotArchiveCleanupLimits, + validateSnapshotArchiveCancellation +} from './SnapshotArchiveCleanup' +import { SnapshotArchiveTransportFailure } from './SnapshotArchiveTransportFailure' +import { getEventListeners } from 'node:events' + +const requestId = 'a'.repeat(64) +const pending = () => ({ version: 1, outcome: 'cleanup-pending', requestId }) +function gate() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} + +afterEach(() => { + jest.restoreAllMocks() + jest.useRealTimers() +}) + +test('only a true cancellation acknowledgement proves completed cleanup', () => { + expect(validateSnapshotArchiveCancellation(true, requestId)).toBeUndefined() + expect(() => validateSnapshotArchiveCancellation(pending(), requestId)).toThrow(SnapshotArchiveCleanupPendingError) +}) + +test.each([ + undefined, + null, + false, + 1, + 'true', + [], + {}, + { ...pending(), version: 2 }, + { ...pending(), outcome: 'closed' }, + { ...pending(), requestId: 'b'.repeat(64) }, + { version: 1, outcome: 'cleanup-pending', other: requestId }, + { ...pending(), extra: true }, + { ...pending(), [Symbol('extra')]: true }, + Object.create(pending()), + Object.defineProperty({ ...pending() }, 'version', { value: 1, enumerable: false }) +])('malformed or foreign cancellation receipt %p cannot acknowledge cleanup', value => { + expect(() => validateSnapshotArchiveCancellation(value, requestId)).toThrow( + new TypeError('Invalid snapshot archive cancellation receipt') + ) +}) + +test('cancellation receipt validation never invokes an accessor', () => { + const getter = jest.fn(() => requestId) + const value = Object.defineProperty({ version: 1, outcome: 'cleanup-pending' }, 'requestId', { + get: getter, + enumerable: true + }) + expect(() => validateSnapshotArchiveCancellation(value, requestId)).toThrow(TypeError) + expect(getter).not.toHaveBeenCalled() +}) + +test('pending cleanup uses bounded exponential delay and one signal until a real acknowledgement', async () => { + jest.useFakeTimers({ now: 1000000 }) + const calls: number[] = [] + const signals: AbortSignal[] = [] + const cancel = jest.fn(async (signal: AbortSignal) => { + expect(getEventListeners(signal, 'abort')).toHaveLength(0) + calls.push(Date.now()) + signals.push(signal) + if (calls.length < 6) throw new SnapshotArchiveCleanupPendingError() + }) + let completed = false + const closing = drainSnapshotArchiveRequest(cancel) + void closing.then( + () => { + completed = true + }, + () => undefined + ) + await jest.advanceTimersByTimeAsync(2499) + expect(completed).toBe(false) + expect(calls).toEqual([1000000, 1000100, 1000300, 1000700, 1001500]) + await jest.advanceTimersByTimeAsync(1) + await closing + expect(calls).toEqual([1000000, 1000100, 1000300, 1000700, 1001500, 1002500]) + expect(new Set(signals).size).toBe(1) + expect(signals[0].aborted).toBe(false) + expect(getEventListeners(signals[0], 'abort')).toHaveLength(0) + expect(jest.getTimerCount()).toBe(0) +}) + +test('repeated pending receipts terminate at the fixed attempt bound without claiming completion', async () => { + jest.useFakeTimers() + const cancel = jest.fn(async () => { + throw new SnapshotArchiveCleanupPendingError() + }) + const closing = drainSnapshotArchiveRequest(cancel) + const outcome = expect(closing).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + await jest.advanceTimersByTimeAsync(30000) + await outcome + expect(snapshotArchiveCleanupLimits).toEqual({ lifetimeMs: 30000, attempts: 32, delayMs: 1000 }) + expect(Object.isFrozen(snapshotArchiveCleanupLimits)).toBe(true) + expect(cancel).toHaveBeenCalledTimes(32) + expect(jest.getTimerCount()).toBe(0) +}) + +test.each(['pending', 'complete', 'failure'] as const)( + 'cleanup deadline aborts I/O but awaits its %s settlement', + async settlement => { + jest.useFakeTimers() + const entered = gate() + const release = gate() + const failure = new Error('synthetic independent cleanup failure') + let signal: AbortSignal | undefined + let settled = false + const cancel = jest.fn(async (current: AbortSignal) => { + signal = current + entered.resolve() + await release.promise + if (settlement === 'pending') throw new SnapshotArchiveCleanupPendingError() + if (settlement === 'failure') throw failure + }) + const closing = drainSnapshotArchiveRequest(cancel) + const observed = closing.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + try { + await entered.promise + expect(signal!.aborted).toBe(false) + await jest.advanceTimersByTimeAsync(30000) + expect(signal!.aborted).toBe(true) + expect(settled).toBe(false) + release.resolve() + if (settlement === 'complete') await expect(closing).resolves.toBeUndefined() + else if (settlement === 'failure') await expect(closing).rejects.toBe(failure) + else await expect(closing).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect(cancel).toHaveBeenCalledTimes(1) + await observed + expect(jest.getTimerCount()).toBe(0) + } finally { + release.resolve() + await observed + } + } +) + +test.each(['wall', 'monotonic'] as const)('stalled timers cannot extend the %s cleanup deadline', async clock => { + jest.useFakeTimers({ now: 1000000 }) + const monotonic = jest.spyOn(performance, 'now').mockReturnValue(4000) + const cancel = jest.fn(async () => { + if (clock === 'wall') jest.setSystemTime(1030000) + else { + jest.setSystemTime(-1000000) + monotonic.mockReturnValue(34000) + } + throw new SnapshotArchiveCleanupPendingError() + }) + await expect(drainSnapshotArchiveRequest(cancel)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect(cancel).toHaveBeenCalledTimes(1) + expect(jest.getTimerCount()).toBe(0) +}) + +test('only native transport loss retries, with the identical cleanup signal', async () => { + jest.useFakeTimers() + const signals: AbortSignal[] = [] + const loss = new SnapshotArchiveTransportFailure('synthetic lost cancellation acknowledgement') + const cancel = jest.fn(async (signal: AbortSignal) => { + signals.push(signal) + throw loss + }) + await expect(drainSnapshotArchiveRequest(cancel)).rejects.toBe(loss) + expect(cancel).toHaveBeenCalledTimes(2) + expect(signals[0]).toBe(signals[1]) + expect(jest.getTimerCount()).toBe(0) +}) + +test('a native failure at the deadline cannot start its retry', async () => { + jest.useFakeTimers({ now: 1000000 }) + const cancel = jest.fn(async () => { + jest.setSystemTime(1030000) + throw new SnapshotArchiveTransportFailure('synthetic deadline loss') + }) + await expect(drainSnapshotArchiveRequest(cancel)).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + expect(cancel).toHaveBeenCalledTimes(1) + expect(jest.getTimerCount()).toBe(0) +}) + +test('timeout during pending backoff clears the delay and never starts another request', async () => { + jest.useFakeTimers() + const cancel = jest.fn(async () => { + await new Promise(resolve => setTimeout(resolve, 29950)) + throw new SnapshotArchiveCleanupPendingError() + }) + const closing = drainSnapshotArchiveRequest(cancel) + const rejected = expect(closing).rejects.toBeInstanceOf(SnapshotArchiveCleanupPendingError) + await jest.advanceTimersByTimeAsync(30000) + await rejected + expect(cancel).toHaveBeenCalledTimes(1) + expect(jest.getTimerCount()).toBe(0) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotLease.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotLease.test.ts new file mode 100644 index 000000000..d76f77d7a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotLease.test.ts @@ -0,0 +1,310 @@ +import { RemoteSnapshotLease } from './RemoteSnapshotLease' +import { remoteReaderFixture } from '../../../../test/utils/remoteSnapshotReaderFixtures' +import { snapshotArchiveReaderRequestId } from './SnapshotArchiveReaderRequest' +import { SnapshotCancelledError } from '../SnapshotCancelledError' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { SnapshotArchiveTransportFailure } from './SnapshotArchiveTransportFailure' + +function request() { + const fields = { version: 2 as const, nonce: 'a'.repeat(64), notAfter: Date.now() + 300000, maxBytes: 32768 } + return { ...fields, requestId: snapshotArchiveReaderRequestId(fields) } +} +function gate() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} +afterEach(() => { + jest.restoreAllMocks() + jest.useRealTimers() +}) + +test.each([0, -1, 3600001, 1.5, NaN, Infinity])('rejects invalid lifetime %p before retaining a timer', lifetimeMs => { + jest.useFakeTimers() + const { transport } = remoteReaderFixture([]) + expect(() => new RemoteSnapshotLease(transport, { lifetimeMs })).toThrow('an integer from 1 to 3600000') + expect(jest.getTimerCount()).toBe(0) +}) + +test('server time includes the entire offer round trip and wall-clock rollback cannot extend the lease', async () => { + jest.useFakeTimers({ now: 1790812800000 }) + const { transport } = remoteReaderFixture([]) + const lease = new RemoteSnapshotLease(transport, { lifetimeMs: 1000 }) + expect(() => lease.now()).toThrow('clock is not bound') + await jest.advanceTimersByTimeAsync(200) + lease.bindServerTime(100000) + expect(lease.now()).toBe(100200) + expect(() => lease.bindServerTime(100001)).toThrow('clock is already bound') + expect(lease.expiresAt).toBe(1790812801000) + jest.setSystemTime(1790812800000 - 3600000) + await jest.advanceTimersByTimeAsync(800) + expect(lease.isOpen).toBe(false) + expect(() => lease.now()).toThrow('expired') + await lease.closed + expect(jest.getTimerCount()).toBe(0) +}) + +test('close fences synchronously, drains the client operation and cancels the exact detached immutable tuple', async () => { + const { transport, rpc } = remoteReaderFixture([]) + const lease = new RemoteSnapshotLease(transport, {}) + const original = request() + const expected = { ...original } + lease.own(original) + original.nonce = 'b'.repeat(64) + expect(() => lease.own(request())).toThrow('already owned') + const entered = gate() + const resume = gate() + let signal: AbortSignal | undefined + const pending = lease.run(async current => { + signal = current + entered.resolve() + await resume.promise + return 1 + }) + void pending.catch(() => undefined) + try { + await entered.promise + await expect(lease.run(async () => 2)).rejects.toThrow('already has an operation') + let closed = false + const closing = lease.close() + void closing.then( + () => { + closed = true + }, + () => undefined + ) + expect(lease.close()).toBe(closing) + expect(lease.isOpen).toBe(false) + expect(signal!.aborted).toBe(true) + await Promise.resolve() + expect(closed).toBe(false) + expect(rpc).not.toHaveBeenCalled() + resume.resolve() + await expect(pending).rejects.toThrow('closed') + await closing + expect(closed).toBe(true) + expect(rpc).toHaveBeenCalledWith( + 'cancelSnapshotArchiveRequest', + [expect.objectContaining({ request: expected })], + expect.any(AbortSignal) + ) + expect(rpc).toHaveBeenCalledTimes(1) + } finally { + resume.resolve() + await pending.catch(() => undefined) + await lease.close() + } +}) + +test('cancellation during polling drains the wait and clears both deadline and delay timers', async () => { + jest.useFakeTimers() + const { transport, rpc } = remoteReaderFixture([]) + const controller = new AbortController() + const lease = new RemoteSnapshotLease(transport, { signal: controller.signal }) + lease.own(request()) + const pending = lease.wait(1000) + void pending.catch(() => undefined) + await Promise.resolve() + controller.abort() + await expect(pending).rejects.toThrow('cancelled') + await lease.closed + expect(jest.getTimerCount()).toBe(0) + expect(rpc).toHaveBeenCalledTimes(1) + expect(lease.isOpen).toBe(false) +}) + +test('failed remote cancellation remains observable through close and closed', async () => { + const { transport } = remoteReaderFixture([]) + const error = new Error('synthetic durable cancellation failure') + jest.spyOn(transport, 'cancelRequest').mockRejectedValue(error) + const lease = new RemoteSnapshotLease(transport, {}) + lease.own(request()) + const closing = lease.close() + await expect(closing).rejects.toBe(error) + await expect(lease.closed).rejects.toBe(error) + expect(lease.close()).toBe(closing) + expect(lease.isOpen).toBe(false) +}) + +test('already-aborted acquisition never starts a request or leaves a timer behind', async () => { + jest.useFakeTimers() + const { transport, rpc } = remoteReaderFixture([]) + const controller = new AbortController() + controller.abort() + const lease = new RemoteSnapshotLease(transport, { signal: controller.signal }) + await expect(lease.run(async () => 1)).rejects.toThrow('cancelled') + await lease.closed + expect(rpc).not.toHaveBeenCalled() + expect(jest.getTimerCount()).toBe(0) +}) + +test.each(['native', 'sdk', 'validation'] as const)( + 'lease expiry classifies %s transport settlement without suppressing independent validation failures', + async kind => { + jest.useFakeTimers() + const { transport } = remoteReaderFixture([]) + const lease = new RemoteSnapshotLease(transport, { lifetimeMs: 10 }) + const entered = gate() + const resume = gate() + const failure = + kind === 'native' + ? new SnapshotArchiveTransportFailure(new Error('synthetic connection loss')) + : kind === 'sdk' + ? new SnapshotCancelledError('synthetic authenticated request cancelled') + : new Error('synthetic authentication failure') + const pending = lease.run(async () => { + entered.resolve() + await resume.promise + throw failure + }) + void pending.catch(() => undefined) + try { + await entered.promise + await jest.advanceTimersByTimeAsync(10) + expect(lease.isOpen).toBe(false) + resume.resolve() + if (kind === 'validation') await expect(pending).rejects.toBe(failure) + else await expect(pending).rejects.toBeInstanceOf(SnapshotResourceLimitError) + await lease.closed + expect(jest.getTimerCount()).toBe(0) + } finally { + resume.resolve() + await pending.catch(() => undefined) + await lease.close() + } + } +) + +test('immutable operation recovery retains one slot, signal and fixed deadline', async () => { + const { transport } = remoteReaderFixture([]) + const lease = new RemoteSnapshotLease(transport, {}) + const entered = gate() + const resume = gate() + const signals: AbortSignal[] = [] + const expiresAt = lease.expiresAt + const operation = jest.fn(async (signal: AbortSignal) => { + signals.push(signal) + if (signals.length === 1) throw new SnapshotArchiveTransportFailure('synthetic loss') + entered.resolve() + await resume.promise + return 7 + }) + const pending = lease.runIdempotent(operation) + try { + await entered.promise + expect(operation).toHaveBeenCalledTimes(2) + expect(signals[0]).toBe(signals[1]) + expect(signals[0].aborted).toBe(false) + expect(lease.expiresAt).toBe(expiresAt) + await expect(lease.run(async () => 2)).rejects.toThrow('already has an operation') + resume.resolve() + await expect(pending).resolves.toBe(7) + } finally { + resume.resolve() + await pending.catch(() => undefined) + await lease.close() + } +}) + +test.each(['cancelled', 'expired'] as const)('a %s lease never starts its connection-loss retry', async kind => { + jest.useFakeTimers() + const { transport } = remoteReaderFixture([]) + const controller = new AbortController() + const lease = new RemoteSnapshotLease(transport, { lifetimeMs: 10, signal: controller.signal }) + const operation = jest.fn(async () => { + if (kind === 'cancelled') controller.abort() + else jest.setSystemTime(Date.now() + 10) + throw new SnapshotArchiveTransportFailure('synthetic loss') + }) + await expect(lease.runIdempotent(operation)).rejects.toThrow(kind) + expect(operation).toHaveBeenCalledTimes(1) + await lease.closed + expect(jest.getTimerCount()).toBe(0) +}) + +test('cleanup retries an identical cancellation once and retains a second transport failure', async () => { + const { transport } = remoteReaderFixture([]) + const failure = new SnapshotArchiveTransportFailure('synthetic repeated loss') + const cancel = jest.spyOn(transport, 'cancelRequest').mockRejectedValue(failure) + const lease = new RemoteSnapshotLease(transport, {}) + lease.own(request()) + await expect(lease.close()).rejects.toBe(failure) + await expect(lease.closed).rejects.toBe(failure) + expect(cancel).toHaveBeenCalledTimes(2) + expect(cancel.mock.calls[0][0]).toBe(cancel.mock.calls[1][0]) +}) + +test.each(['timer', 'abort', 'deadline-check'] as const)( + '%s automatic closure retains a failed cancellation receipt for its observer', + async reason => { + jest.useFakeTimers() + const { transport } = remoteReaderFixture([]) + const controller = new AbortController() + const failure = new Error('synthetic cancellation receipt failure') + const cancel = jest.spyOn(transport, 'cancelRequest').mockRejectedValue(failure) + const lease = new RemoteSnapshotLease(transport, { lifetimeMs: 10, signal: controller.signal }) + lease.own(request()) + if (reason === 'timer') await jest.advanceTimersByTimeAsync(10) + else if (reason === 'abort') controller.abort() + else { + jest.setSystemTime(Date.now() + 10) + expect(() => lease.assertOpen()).toThrow('expired') + } + await expect(lease.closed).rejects.toBe(failure) + await expect(lease.close()).rejects.toBe(failure) + expect(cancel).toHaveBeenCalledTimes(1) + expect(jest.getTimerCount()).toBe(0) + } +) + +test.each([1, 3600000])('inclusive lifetime %p retains its exact deadline', async lifetimeMs => { + jest.useFakeTimers({ now: 1790812800000 }) + const { transport } = remoteReaderFixture([]) + const lease = new RemoteSnapshotLease(transport, { lifetimeMs }) + expect(lease.lifetimeMs).toBe(lifetimeMs) + expect(lease.expiresAt).toBe(1790812800000 + lifetimeMs) + expect(lease.isOpen).toBe(true) + await lease.close() + expect(jest.getTimerCount()).toBe(0) +}) + +test('a nonzero monotonic origin and stalled timers cannot extend the fixed server clock or deadline', async () => { + jest.useFakeTimers({ now: 1790812800000 }) + const monotonic = jest.spyOn(performance, 'now').mockReturnValue(4000) + const { transport } = remoteReaderFixture([]) + const lease = new RemoteSnapshotLease(transport, { lifetimeMs: 1000 }) + monotonic.mockReturnValue(4125) + lease.bindServerTime(100000) + expect(lease.now()).toBe(100125) + monotonic.mockReturnValue(4999) + expect(lease.now()).toBe(100999) + jest.setSystemTime(1790812800000 - 3600000) + monotonic.mockReturnValue(5000) + expect(() => lease.assertOpen()).toThrow('expired') + await lease.closed + expect(jest.getTimerCount()).toBe(0) +}) + +test('poll delay holds the operation until its exact duration without issuing remote work', async () => { + jest.useFakeTimers() + const { transport, rpc } = remoteReaderFixture([]) + const lease = new RemoteSnapshotLease(transport, { lifetimeMs: 1000 }) + let finished = false + const waiting = lease.wait(100) + void waiting.then( + () => { + finished = true + }, + () => undefined + ) + await jest.advanceTimersByTimeAsync(99) + expect(finished).toBe(false) + await jest.advanceTimersByTimeAsync(1) + await waiting + expect(finished).toBe(true) + expect(rpc).not.toHaveBeenCalled() + await lease.close() + expect(jest.getTimerCount()).toBe(0) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotLease.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotLease.ts new file mode 100644 index 000000000..189827144 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotLease.ts @@ -0,0 +1,176 @@ +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { SnapshotCancelledError } from '../SnapshotCancelledError' +import { isSnapshotArchiveTransportFailure, retrySnapshotArchiveOperation } from './SnapshotArchiveTransportFailure' +import type { WalletReadSnapshotOptions } from '../WalletReadSnapshot' +import { parseSnapshotArchiveReaderRequest, type SnapshotArchiveReaderRequest } from './SnapshotArchiveReaderRequest' +import type { SnapshotArchiveTransport } from './SnapshotArchiveTransport' +import { drainSnapshotArchiveRequest } from './SnapshotArchiveCleanup' + +function observedCompletion() { + let resolve!: () => void + let reject!: (error: unknown) => void + const closed = new Promise((onResolve, onReject) => { + resolve = onResolve + reject = onReject + }) + void closed.catch(() => undefined) + return { closed, resolve, reject } +} + +/** Owns one opening/read operation and one expiry timer until remote cleanup settles. */ +export class RemoteSnapshotLease { + readonly lifetimeMs: number + readonly expiresAt: number + readonly closed: Promise + private readonly startedAt = performance.now() + private readonly controller = new AbortController() + private readonly signal: AbortSignal | undefined + private readonly timer: ReturnType | undefined + private pending: Promise | undefined + private request: Readonly | undefined + private serverTime: number | undefined + private reason: WERR_INVALID_OPERATION | undefined + private closing: Promise | undefined + private readonly resolveClosed: () => void + private readonly rejectClosed: (error: unknown) => void + + constructor( + private readonly transport: SnapshotArchiveTransport, + options: WalletReadSnapshotOptions + ) { + this.lifetimeMs = options.lifetimeMs ?? 300000 + if (!Number.isSafeInteger(this.lifetimeMs) || this.lifetimeMs < 1 || this.lifetimeMs > 3600000) + throw new WERR_INVALID_PARAMETER('lifetimeMs', 'an integer from 1 to 3600000') + this.expiresAt = Date.now() + this.lifetimeMs + this.signal = options.signal + const completion = observedCompletion() + this.closed = completion.closed + this.resolveClosed = completion.resolve + this.rejectClosed = completion.reject + this.signal?.addEventListener('abort', this.abort, { once: true }) + if (this.signal?.aborted === true) this.abort() + else + this.timer = setTimeout(() => { + void this.close(new SnapshotResourceLimitError('Remote snapshot expired')).catch(() => undefined) + }, this.lifetimeMs) + } + + private readonly abort = (): void => { + void this.close(new SnapshotCancelledError('Remote snapshot was cancelled')).catch(() => undefined) + } + + get isOpen(): boolean { + try { + this.assertOpen() + return true + } catch { + return false + } + } + + assertOpen(): void { + if (Date.now() >= this.expiresAt || performance.now() - this.startedAt >= this.lifetimeMs) + void this.close(new SnapshotResourceLimitError('Remote snapshot expired')).catch(() => undefined) + if (this.reason !== undefined) throw this.reason + } + + bindServerTime(serverTime: number): void { + this.assertOpen() + if (this.serverTime !== undefined) throw new WERR_INVALID_OPERATION('Remote snapshot clock is already bound') + this.serverTime = serverTime + } + + /** Includes the entire offer RTT conservatively; retries never renew this basis. */ + now(): number { + this.assertOpen() + if (this.serverTime === undefined) throw new WERR_INVALID_OPERATION('Remote snapshot clock is not bound') + return Math.ceil(this.serverTime + performance.now() - this.startedAt) + } + + own(request: SnapshotArchiveReaderRequest): void { + this.assertOpen() + if (this.request !== undefined) throw new WERR_INVALID_OPERATION('Remote snapshot request is already owned') + this.request = parseSnapshotArchiveReaderRequest(request) + } + + async run(operation: (signal: AbortSignal) => Promise): Promise { + this.assertOpen() + if (this.pending !== undefined) + throw new WERR_INVALID_OPERATION('Remote snapshot already has an operation in flight') + const pending = Promise.resolve().then(async () => { + this.assertOpen() + return await operation(this.controller.signal) + }) + // Own settlement immediately, separately from the result returned to the + // caller. Cleanup drains failures without replacing their public identity. + this.pending = pending.then( + () => undefined, + () => undefined + ) + try { + const value = await pending + this.assertOpen() + return value + } catch (error) { + // Only a local transport cancellation may be explained by our abort. + // Authentication, framing and cleanup failures retain their own identity. + if ( + this.reason !== undefined && + (error instanceof SnapshotCancelledError || isSnapshotArchiveTransportFailure(error)) + ) + throw this.reason + throw error + } finally { + this.pending = undefined + } + } + + /** Keep one operation slot and the fixed lease across a single connection-loss retry. */ + async runIdempotent(operation: (signal: AbortSignal) => Promise): Promise { + return await this.run(async signal => + retrySnapshotArchiveOperation(async () => { + this.assertOpen() + return await operation(signal) + }) + ) + } + + async wait(milliseconds: number): Promise { + await this.run( + async signal => + await new Promise(resolve => { + const finish = (): void => { + signal.removeEventListener('abort', abort) + resolve() + } + const timer = setTimeout(finish, milliseconds) + const abort = (): void => { + clearTimeout(timer) + // run() checks the recorded close reason after this wait settles. + finish() + } + // run() asserted openness immediately before this synchronous setup. + signal.addEventListener('abort', abort, { once: true }) + }) + ) + } + + close(reason = new WERR_INVALID_OPERATION('Remote snapshot is closed')): Promise { + if (this.closing !== undefined) return this.closing + this.reason = reason + if (this.timer !== undefined) clearTimeout(this.timer) + this.signal?.removeEventListener('abort', this.abort) + const pending = this.pending + // Fence first, then drain the network operation before releasing its shared receipt. + this.closing = Promise.resolve().then(async () => { + await pending + const request = this.request + if (request !== undefined) + await drainSnapshotArchiveRequest(signal => this.transport.cancelRequest(request, signal)) + }) + void this.closing.then(this.resolveClosed, this.rejectClosed) + this.controller.abort() + return this.closing + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotOpening.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotOpening.test.ts new file mode 100644 index 000000000..ccfa3bf84 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotOpening.test.ts @@ -0,0 +1,211 @@ +import { remoteReaderFixture } from '../../../../test/utils/remoteSnapshotReaderFixtures' +import { openRemoteSnapshot, RemoteSnapshotOpeningCleanupError } from './openRemoteSnapshot' +import { SnapshotArchiveTransportFailure } from './SnapshotArchiveTransportFailure' +import { SnapshotArchiveTransport } from './SnapshotArchiveTransport' + +afterEach(() => jest.restoreAllMocks()) + +test('an old server is declined before any request or timer is retained', async () => { + jest.useFakeTimers() + try { + const rpc = jest.fn() + const transport = new SnapshotArchiveTransport(rpc, '02' + '11'.repeat(32), 'source', 'test') + expect(await openRemoteSnapshot(transport)).toBeUndefined() + expect(rpc).not.toHaveBeenCalled() + expect(jest.getTimerCount()).toBe(0) + } finally { + jest.useRealTimers() + } +}) + +test('a refused server offer declines before any capture request exists', async () => { + const { transport, rpc } = remoteReaderFixture([]) + jest.spyOn(transport, 'readerOffer').mockResolvedValue({ version: 1, outcome: 'resource-limited' }) + expect(await openRemoteSnapshot(transport)).toBeUndefined() + expect(rpc).not.toHaveBeenCalled() +}) + +test.each(['before', 'after'])( + 'one same-request retry recovers an admission response lost %s the server accepted it', + async when => { + const { transport, rpc } = remoteReaderFixture([]) + const implementation = transport.admit.bind(transport) + const admitted = jest.spyOn(transport, 'admit') + admitted.mockImplementationOnce(async (request, signal) => { + if (when === 'after') await implementation(request, signal) + throw new SnapshotArchiveTransportFailure(new Error('synthetic lost response')) + }) + const view = (await openRemoteSnapshot(transport))! + try { + expect(admitted).toHaveBeenCalledTimes(2) + expect(admitted.mock.calls[1][0]).toEqual(admitted.mock.calls[0][0]) + expect(rpc.mock.calls.filter(([method]) => method === 'getSnapshotArchiveReaderOffer')).toHaveLength(1) + expect(rpc.mock.calls.filter(([method]) => method === 'getSnapshotArchiveStatus')).toHaveLength(0) + expect(view.isOpen).toBe(true) + } finally { + await view.close() + } + } +) + +test('an admission refusal closes its retained offer before permitting fallback', async () => { + const { transport, rpc } = remoteReaderFixture([]) + const admitted = jest.spyOn(transport, 'admit').mockImplementation(async request => ({ + version: 1, + outcome: 'resource-limited', + requestId: request.requestId, + expiresAt: request.notAfter + })) + expect(await openRemoteSnapshot(transport)).toBeUndefined() + const request = admitted.mock.calls[0][0] + expect(rpc.mock.calls.map(([method]) => method)).toEqual([ + 'getSnapshotArchiveReaderOffer', + 'cancelSnapshotArchiveRequest' + ]) + expect(rpc.mock.calls[1][1]).toEqual([{ version: 1, identityKey: expect.any(String), request }]) +}) + +test('unknown admission errors never trigger retries, status recovery or compatibility fallback', async () => { + const { transport, rpc } = remoteReaderFixture([]) + const failure = new Error('synthetic signature or framing validation failure') + const admitted = jest.spyOn(transport, 'admit').mockRejectedValue(failure) + await expect(openRemoteSnapshot(transport)).rejects.toBe(failure) + expect(admitted).toHaveBeenCalledTimes(1) + expect(rpc.mock.calls.map(([method]) => method)).toEqual([ + 'getSnapshotArchiveReaderOffer', + 'cancelSnapshotArchiveRequest' + ]) +}) + +test('a second native-fetch failure is bounded and still cancels the identical request', async () => { + const { transport, rpc } = remoteReaderFixture([]) + const failure = new SnapshotArchiveTransportFailure(new Error('synthetic repeated connection loss')) + const admitted = jest.spyOn(transport, 'admit').mockRejectedValue(failure) + await expect(openRemoteSnapshot(transport)).rejects.toBe(failure) + expect(admitted).toHaveBeenCalledTimes(2) + expect(admitted.mock.calls[0][0]).toEqual(admitted.mock.calls[1][0]) + expect(rpc.mock.calls.map(([method]) => method)).toEqual([ + 'getSnapshotArchiveReaderOffer', + 'cancelSnapshotArchiveRequest' + ]) +}) + +test('opening and cancellation failures both remain observable without requiring AggregateError', async () => { + const { transport } = remoteReaderFixture([]) + const original = new Error('synthetic malformed archive') + const cleanup = new Error('synthetic failed cancellation acknowledgement') + jest.spyOn(transport, 'directory').mockRejectedValue(original) + jest.spyOn(transport, 'cancelRequest').mockRejectedValue(cleanup) + const error = await openRemoteSnapshot(transport).catch(error => error) + expect(error).toBeInstanceOf(RemoteSnapshotOpeningCleanupError) + expect(error.cause).toBe(original) + expect(error.cleanupError).toBe(cleanup) +}) + +test('status recovery uses the admitted tuple without requesting another offer or capture', async () => { + const { transport, rpc } = remoteReaderFixture([]) + const admit = transport.admit.bind(transport) + jest.spyOn(transport, 'admit').mockImplementation(async (...args) => { + const admission = await admit(...args) + if (admission.outcome !== 'accepted') throw new Error('Expected fixture admission') + return { + ...admission, + receipt: { ...admission.receipt, state: 'building', archiveId: undefined, digest: undefined } + } + }) + const status = jest.spyOn(transport, 'readerStatus') + status.mockRejectedValueOnce(new SnapshotArchiveTransportFailure('synthetic status connection loss')) + const view = (await openRemoteSnapshot(transport))! + try { + expect(status).toHaveBeenCalledTimes(2) + expect(status.mock.calls[0]).toEqual(status.mock.calls[1]) + expect(rpc.mock.calls.filter(([method]) => method === 'getSnapshotArchiveReaderOffer')).toHaveLength(1) + expect(rpc.mock.calls.filter(([method]) => method === 'admitSnapshotArchive')).toHaveLength(1) + } finally { + await view.close() + } +}) + +test.each(['resource-limited', 'failed', 'closed', 'expired'] as const)( + 'a durable %s capture receipt is preserved until exact cancellation completes', + async state => { + const { transport, rpc } = remoteReaderFixture([]) + const implementation = transport.admit.bind(transport) + jest.spyOn(transport, 'admit').mockImplementation(async (...args) => { + const admitted = await implementation(...args) + if (admitted.outcome !== 'accepted') throw new Error('Expected fixture admission') + return { + ...admitted, + receipt: { + version: 1, + requestId: admitted.receipt.requestId, + expiresAt: admitted.receipt.expiresAt, + state + } + } + }) + const pending = openRemoteSnapshot(transport) + if (state === 'resource-limited') await expect(pending).resolves.toBeUndefined() + else await expect(pending).rejects.toThrow(`Snapshot archive capture is ${state}`) + expect(rpc.mock.calls.map(([method]) => method)).toEqual([ + 'getSnapshotArchiveReaderOffer', + 'admitSnapshotArchive', + 'cancelSnapshotArchiveRequest' + ]) + } +) + +test('directory verification cannot expose a view at the conservatively rounded server expiry', async () => { + jest.useFakeTimers() + const monotonic = jest.spyOn(performance, 'now').mockReturnValue(0) + const { transport, rpc } = remoteReaderFixture([]) + const directory = transport.directory.bind(transport) + jest.spyOn(transport, 'directory').mockImplementation(async (...args) => { + const result = await directory(...args) + monotonic.mockReturnValue(999.1) + return result + }) + try { + await expect(openRemoteSnapshot(transport, { lifetimeMs: 1000 })).rejects.toThrow('Remote snapshot expired') + expect(rpc.mock.calls.filter(([method]) => method === 'cancelSnapshotArchiveRequest')).toHaveLength(1) + expect(jest.getTimerCount()).toBe(0) + } finally { + jest.useRealTimers() + } +}) + +test('building status uses bounded exponential polling without renewing or readmitting its request', async () => { + jest.useFakeTimers({ now: 1790812800000 }) + const controller = new AbortController() + const { transport, rpc } = remoteReaderFixture([]) + const originalAdmit = transport.admit.bind(transport) + const originalStatus = transport.readerStatus.bind(transport) + const times: number[] = [] + jest.spyOn(transport, 'admit').mockImplementation(async (...args) => { + const admitted = await originalAdmit(...args) + if (admitted.outcome !== 'accepted') throw new Error('Expected accepted fixture') + return { ...admitted, receipt: { ...admitted.receipt, state: 'building' } } + }) + const status = jest.spyOn(transport, 'readerStatus').mockImplementation(async (...args) => { + times.push(Date.now() - 1790812800000) + const receipt = await originalStatus(...args) + return times.length < 6 ? { ...receipt, state: 'building' } : receipt + }) + const pending = openRemoteSnapshot(transport, { signal: controller.signal }) + void pending.catch(() => undefined) + try { + await jest.advanceTimersByTimeAsync(3500) + expect(times).toEqual([100, 300, 700, 1500, 2500, 3500]) + const view = await pending + expect(view!.isOpen).toBe(true) + expect(view!.expiresAt).toBe(1790813100000) + expect(status.mock.calls.every(call => call[0] === status.mock.calls[0][0])).toBe(true) + expect(rpc.mock.calls.filter(([method]) => method === 'getSnapshotArchiveReaderOffer')).toHaveLength(1) + expect(rpc.mock.calls.filter(([method]) => method === 'admitSnapshotArchive')).toHaveLength(1) + } finally { + controller.abort() + const view = await pending.catch(() => undefined) + await view?.close() + jest.useRealTimers() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotPageReader.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotPageReader.ts new file mode 100644 index 000000000..c784108d2 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotPageReader.ts @@ -0,0 +1,226 @@ +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import type { + PackedSnapshotRow, + WalletReadSnapshot, + WalletSnapshotCursor, + WalletSnapshotPage, + WalletSnapshotPageLimits, + WalletSnapshotTable, + WalletSnapshotTables +} from '../WalletReadSnapshot' +import type { VerifiedSnapshotArchiveDirectory } from './SnapshotArchiveDirectory' +import type { SnapshotArchiveTransport } from './SnapshotArchiveTransport' +import type { RemoteSnapshotLease } from './RemoteSnapshotLease' +import { + decodeRemoteSnapshotFrame, + detachRemoteSnapshotRow, + remoteSnapshotKeys, + type RemoteSnapshotFrame, + type RemoteSnapshotRow +} from './RemoteSnapshotRows' + +interface Position { + sequence: number + rowOffset: number +} +interface DetachedCursor { + cursor: WalletSnapshotCursor + position: Position +} + +function invalidCursor(): never { + throw new WERR_INVALID_PARAMETER('cursor', 'a verified position in this snapshot archive and table') +} +function data(input: unknown, names: string[]): Record { + if ( + input === null || + typeof input !== 'object' || + Array.isArray(input) || + Reflect.ownKeys(input).length !== names.length + ) + invalidCursor() + return Object.fromEntries( + names.map(name => { + const property = Object.getOwnPropertyDescriptor(input, name) + if (property === undefined || !('value' in property) || !property.enumerable) invalidCursor() + return [name, property.value] + }) + ) +} +function detachCursor( + input: WalletSnapshotCursor | undefined, + table: WalletSnapshotTable, + directory: VerifiedSnapshotArchiveDirectory +): DetachedCursor | undefined { + if (input === undefined) return undefined + const cursor = data(input, ['version', 'snapshotId', 'table', 'after', 'archivePosition']) + const position = data(cursor.archivePosition, ['version', 'archiveId', 'sequence', 'rowOffset']) + const keys = remoteSnapshotKeys(table) + const range = directory.tables[table] + if ( + cursor.version !== 1 || + cursor.snapshotId !== directory.manifest.binding.snapshotId || + cursor.table !== table || + !Array.isArray(cursor.after) || + cursor.after.length !== keys.length || + position.version !== 1 || + position.archiveId !== directory.manifest.archiveId || + !Number.isSafeInteger(position.sequence) || + (position.sequence as number) < range.first || + (position.sequence as number) >= range.first + range.pages || + !Number.isSafeInteger(position.rowOffset) || + (position.rowOffset as number) < 1 + ) + invalidCursor() + const sequence = position.sequence as number + const rowOffset = position.rowOffset as number + if (rowOffset > directory.receipts[sequence].rows) invalidCursor() + const after = keys.map((key, index) => { + const property = Object.getOwnPropertyDescriptor(cursor.after, index) + if (property === undefined || !('value' in property)) invalidCursor() + const value: unknown = property.value + if ( + key === 'fieldName' + ? typeof value !== 'string' || value.length > 200 || Array.from(value).length > 100 + : !Number.isSafeInteger(value) || (value as number) < 1 + ) + invalidCursor() + return value as number | string + }) + return { + position: { sequence, rowOffset }, + cursor: { + version: 1, + snapshotId: directory.manifest.binding.snapshotId, + table, + after, + archivePosition: { version: 1, archiveId: directory.manifest.archiveId, sequence, rowOffset } + } + } +} + +function limit(value: number | undefined, fallback: number, ceiling: number, name: string): number { + const result = value ?? fallback + if (!Number.isSafeInteger(result) || result < 1 || result > ceiling) + throw new WERR_INVALID_PARAMETER(name, `an integer from 1 to ${ceiling}`) + return result +} + +function fitsPage(rows: number, payloadBytes: number, charge: number, maxBytes: number): boolean { + if (payloadBytes + charge <= maxBytes) return true + if (rows === 0) + throw new SnapshotResourceLimitError('Snapshot row exceeds maxBytes; large-value streaming is required') + return false +} + +/** One private decoded frame, plus returned rows charged to the caller's allocation budget. */ +export function createRemoteSnapshotPageReader( + transport: SnapshotArchiveTransport, + directory: VerifiedSnapshotArchiveDirectory, + lease: RemoteSnapshotLease +): WalletReadSnapshot['readPage'] { + let cached: { sequence: number; frame: RemoteSnapshotFrame } | undefined + let busy = false + const discard = (): void => { + cached = undefined + } + void lease.closed.then(discard, discard) + const assertActive = (): void => { + lease.assertOpen() + if (lease.now() >= directory.manifest.expiresAt) { + const error = new SnapshotResourceLimitError('Remote snapshot expired') + void lease.close(error).catch(() => undefined) + throw error + } + } + const frameAt = async (sequence: number): Promise => { + assertActive() + if (cached?.sequence === sequence) return cached.frame + discard() + try { + const bytes = await lease.runIdempotent(signal => transport.page(directory, sequence, signal)) + assertActive() + const frame = decodeRemoteSnapshotFrame( + bytes, + directory.receipts[sequence], + directory.manifest.binding.user.userId + ) + cached = { sequence, frame } + return frame + } catch (error) { + void lease.close(new WERR_INVALID_OPERATION('Remote snapshot read failed')).catch(() => undefined) + throw error + } + } + const cursorAfter = ( + table: WalletSnapshotTable, + row: RemoteSnapshotRow, + sequence: number, + rowOffset: number + ): WalletSnapshotCursor => ({ + version: 1, + snapshotId: directory.manifest.binding.snapshotId, + table, + after: remoteSnapshotKeys(table).map(key => row[key] as number | string), + archivePosition: { version: 1, archiveId: directory.manifest.archiveId, sequence, rowOffset } + }) + const verifiedPosition = async ( + table: WalletSnapshotTable, + cursor: DetachedCursor | undefined + ): Promise => { + if (cursor === undefined) return { sequence: directory.tables[table].first, rowOffset: 0 } + const frame = await frameAt(cursor.position.sequence) + const row = frame.rows[cursor.position.rowOffset - 1] + if (cursor.cursor.after.some((value, index) => value !== row[remoteSnapshotKeys(table)[index]])) invalidCursor() + return { ...cursor.position } + } + return async ( + table: T, + cursor?: WalletSnapshotCursor, + limits: WalletSnapshotPageLimits = {} + ): Promise> => { + assertActive() + if (busy) throw new WERR_INVALID_OPERATION('Remote snapshot already has a read in flight') + remoteSnapshotKeys(table) + const initialCursor = detachCursor(cursor, table, directory) + const maxRows = limit(limits.maxRows, 128, 1000, 'maxRows') + const maxBytes = limit(limits.maxBytes, 262144, 16777216, 'maxBytes') + busy = true + try { + const position = await verifiedPosition(table, initialCursor) + const rows: Array> = [] + let payloadBytes = 0 + let nextCursor = initialCursor?.cursor + let done = false + let full = false + while (!done && !full) { + const frame = await frameAt(position.sequence) + const receipt = directory.receipts[position.sequence] + while (position.rowOffset < frame.rows.length && rows.length < maxRows) { + const charge = frame.charges[position.rowOffset] + if (!fitsPage(rows.length, payloadBytes, charge, maxBytes)) { + full = true + break + } + const row = frame.rows[position.rowOffset] + rows.push(detachRemoteSnapshotRow(row) as PackedSnapshotRow) + payloadBytes += charge + position.rowOffset++ + nextCursor = cursorAfter(table, row, position.sequence, position.rowOffset) + } + const consumed = position.rowOffset === frame.rows.length + done = consumed && receipt.done + full ||= rows.length === maxRows + if (consumed && !done) { + position.sequence++ + position.rowOffset = 0 + } + } + assertActive() + return { rows, payloadBytes, cursor: nextCursor, done } + } finally { + busy = false + } + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.property.test.ts new file mode 100644 index 000000000..195c4944c --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.property.test.ts @@ -0,0 +1,116 @@ +import fc from 'fast-check' +import { openRemoteSnapshot } from './openRemoteSnapshot' +import { label, remoteReaderFixture } from '../../../../test/utils/remoteSnapshotReaderFixtures' +import type { WalletSnapshotCursor } from '../WalletReadSnapshot' +import { drainSnapshotArchiveRequest, SnapshotArchiveCleanupPendingError } from './SnapshotArchiveCleanup' +import { SnapshotArchiveTransportFailure } from './SnapshotArchiveTransportFailure' +import { getEventListeners } from 'node:events' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +test('generated pending-cleanup and lost-ack schedules preserve one signal, failure identity and bounded polling', async () => { + jest.useFakeTimers() + try { + await fc.assert( + fc.asyncProperty(fc.array(fc.boolean(), { maxLength: 8 }), fc.boolean(), async (lost, fails) => { + let position = 0 + let retry = false + let calls = 0 + const signals = new Set() + const failure = new Error('synthetic independent cleanup failure') + const closing = drainSnapshotArchiveRequest(async signal => { + calls++ + signals.add(signal) + expect(signal.aborted).toBe(false) + expect(getEventListeners(signal, 'abort')).toHaveLength(0) + if (position < lost.length) { + if (lost[position] && !retry) { + retry = true + throw new SnapshotArchiveTransportFailure('synthetic lost acknowledgement') + } + position++ + retry = false + throw new SnapshotArchiveCleanupPendingError() + } + if (fails) throw failure + }) + const outcome = closing.then( + () => undefined, + error => error + ) + await jest.advanceTimersByTimeAsync(10000) + if (fails) expect(await outcome).toBe(failure) + else expect(await outcome).toBeUndefined() + expect(calls).toBe(1 + lost.length + lost.filter(Boolean).length) + expect(position).toBe(lost.length) + expect(signals.size).toBe(1) + expect(jest.getTimerCount()).toBe(0) + for (const signal of signals) expect(getEventListeners(signal, 'abort')).toHaveLength(0) + }) + ) + } finally { + jest.useRealTimers() + } +}) + +test('generated frame/page boundaries preserve exact rows, final keys, allocation charges and immutable retries', async () => { + await fc.assert( + fc.asyncProperty( + fc.array(fc.string({ maxLength: 40 }), { minLength: 1, maxLength: 64 }), + fc.integer({ min: 1, max: 17 }), + fc.integer({ min: 1, max: 13 }), + async (names, frameRows, maxRows) => { + const expected = names.map((name, index) => label(index + 1, name)) + const fixture = remoteReaderFixture(expected, 'txLabels', frameRows) + const view = (await openRemoteSnapshot(fixture.transport))! + try { + let cursor: WalletSnapshotCursor | undefined + let consumed = 0 + let done = false + while (!done) { + const page = await view.readPage('txLabels', cursor, { maxRows, maxBytes: 65536 }) + expect(page.rows).toEqual(expected.slice(consumed, consumed + maxRows)) + expect(page.payloadBytes).toBe(page.rows.reduce((sum, row) => sum + 384 + row.label.length * 2, 0)) + expect(await view.readPage('txLabels', cursor, { maxRows, maxBytes: 65536 })).toEqual(page) + consumed += page.rows.length + expect(page.cursor!.after).toEqual([consumed]) + expect(page.cursor!.archivePosition).toEqual({ + version: 1, + archiveId: fixture.directory.archiveId, + sequence: 8 + Math.floor((consumed - 1) / frameRows), + rowOffset: ((consumed - 1) % frameRows) + 1 + }) + cursor = page.cursor + done = page.done + expect(done).toBe(consumed === expected.length) + } + expect(consumed).toBe(expected.length) + expect((await view.readPage('txLabels', cursor)).rows).toEqual([]) + const sequenceRequests = fixture.rpc.mock.calls + .filter(([method]) => method === 'readSnapshotArchivePage') + .map(([, params]) => (params[0] as { sequence: number }).sequence) + expect( + sequenceRequests.every(sequence => sequence >= 8 && sequence < 8 + Math.ceil(expected.length / frameRows)) + ).toBe(true) + await expect( + view.readPage('txLabels', { + ...cursor!, + archivePosition: { ...cursor!.archivePosition!, archiveId: 'c'.repeat(64) } + }) + ).rejects.toThrow() + } finally { + await view.close() + } + expect(fixture.rpc.mock.calls.filter(([method]) => method === 'cancelSnapshotArchiveRequest')).toHaveLength(1) + } + ) + ) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.test.ts new file mode 100644 index 000000000..e855cbbd7 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReader.test.ts @@ -0,0 +1,334 @@ +import { openRemoteSnapshot } from './openRemoteSnapshot' +import { label, remoteReaderFixture } from '../../../../test/utils/remoteSnapshotReaderFixtures' +import { snapshotArchiveTables } from './SnapshotArchive' +import type { WalletSnapshotCursor } from '../WalletReadSnapshot' + +afterEach(() => jest.restoreAllMocks()) + +test('verified table positioning slices rows and bytes without downloading earlier tables', async () => { + const rows = Array.from({ length: 35 }, (_, index) => label(index + 1)) + const fixture = remoteReaderFixture(rows, 'txLabels', 17) + const view = (await openRemoteSnapshot(fixture.transport))! + try { + const first = await view.readPage('txLabels', undefined, { maxRows: 5 }) + expect(first.rows).toEqual(rows.slice(0, 5)) + expect(first.done).toBe(false) + expect(first.cursor).toEqual({ + version: 1, + snapshotId: view.snapshotId, + table: 'txLabels', + after: [5], + archivePosition: { version: 1, archiveId: fixture.directory.archiveId, sequence: 8, rowOffset: 5 } + }) + expect(first.payloadBytes).toBe(5 * (6 * 64 + 'label-1'.length * 2)) + expect(await view.readPage('txLabels', undefined, { maxRows: 5 })).toEqual(first) + const collected = [...first.rows] + let page = first + while (!page.done) { + page = await view.readPage('txLabels', page.cursor, { maxRows: 5 }) + collected.push(...page.rows) + } + expect(collected).toEqual(rows) + const requested = fixture.rpc.mock.calls + .filter(([method]) => method === 'readSnapshotArchivePage') + .map(([, params]) => (params[0] as { sequence: number }).sequence) + expect(requested).toEqual([8, 9, 10]) + expect((await view.readPage('txLabels', page.cursor)).rows).toEqual([]) + expect((await view.readPage('txLabels', page.cursor)).done).toBe(true) + expect(view.sourceStorage.storageIdentityKey).toBe('original-storage') + expect(view.user.activeStorage).toBe('historical-primary') + expect(view.isOpen).toBe(true) + } finally { + await view.close() + } + expect(view.isOpen).toBe(false) + await expect(view.closed).resolves.toBeUndefined() + expect(fixture.rpc.mock.calls.filter(([method]) => method === 'cancelSnapshotArchiveRequest')).toHaveLength(1) +}) + +test('every empty table still authenticates and decodes its terminal frame', async () => { + const fixture = remoteReaderFixture([]) + const view = (await openRemoteSnapshot(fixture.transport))! + try { + for (const table of snapshotArchiveTables) + expect(await view.readPage(table)).toEqual({ rows: [], payloadBytes: 0, cursor: undefined, done: true }) + expect(fixture.rpc.mock.calls.filter(([method]) => method === 'readSnapshotArchivePage')).toHaveLength(13) + } finally { + await view.close() + } +}) + +test('returned metadata, dates, rows and cursor objects cannot mutate the private archive cache', async () => { + const fixture = remoteReaderFixture([label(1), label(2)]) + const view = (await openRemoteSnapshot(fixture.transport))! + try { + const expected = await view.readPage('txLabels', undefined, { maxRows: 1 }) + const page = await view.readPage('txLabels', undefined, { maxRows: 1 }) + view.sourceStorage.created_at.setTime(0) + view.user.updated_at.setTime(0) + page.rows[0].created_at.setTime(0) + page.rows[0].label = 'changed by caller' + ;(page.cursor!.after as number[])[0] = 999 + ;(page.cursor!.archivePosition as { rowOffset: number }).rowOffset = 2 + expect(await view.readPage('txLabels', undefined, { maxRows: 1 })).toEqual(expected) + expect(view.sourceStorage.created_at.getTime()).not.toBe(0) + expect(view.user.updated_at.getTime()).not.toBe(0) + expect((await view.readPage('txLabels', expected.cursor, { maxRows: 1 })).rows).toEqual([label(2)]) + } finally { + await view.close() + } +}) + +test('cursor positions must match the actual last key in the authenticated frame', async () => { + const fixture = remoteReaderFixture([label(1), label(2), label(3)], 'txLabels', 2) + const view = (await openRemoteSnapshot(fixture.transport))! + try { + const { cursor } = await view.readPage('txLabels', undefined, { maxRows: 1 }) + const changes = [ + { version: 2 }, + { snapshotId: 'c'.repeat(64) }, + { table: 'outputs' }, + { after: [2] }, + { archivePosition: { ...cursor!.archivePosition, archiveId: 'c'.repeat(64) } }, + { archivePosition: { ...cursor!.archivePosition, sequence: 7 } }, + { archivePosition: { ...cursor!.archivePosition, sequence: 10 } }, + { archivePosition: { ...cursor!.archivePosition, rowOffset: 0 } }, + { archivePosition: { ...cursor!.archivePosition, rowOffset: 3 } }, + { archivePosition: undefined } + ] + for (const change of changes) + await expect(view.readPage('txLabels', { ...cursor!, ...change } as WalletSnapshotCursor)).rejects.toThrow() + expect((await view.readPage('txLabels', cursor)).rows).toEqual([label(2), label(3)]) + } finally { + await view.close() + } +}) + +test('row budgets are inclusive, preserve continuation and never omit an oversized row', async () => { + const fixture = remoteReaderFixture([label(1), label(2)]) + const view = (await openRemoteSnapshot(fixture.transport))! + const charge = 6 * 64 + 'label-1'.length * 2 + try { + await expect(view.readPage('txLabels', undefined, { maxBytes: charge - 1 })).rejects.toThrow('exceeds maxBytes') + const first = await view.readPage('txLabels', undefined, { maxBytes: charge }) + expect(first.rows).toEqual([label(1)]) + expect(first.done).toBe(false) + const last = await view.readPage('txLabels', first.cursor, { maxBytes: charge }) + expect(last.rows).toEqual([label(2)]) + expect(last.done).toBe(true) + expect(last.payloadBytes).toBe(charge) + } finally { + await view.close() + } +}) + +test('certificate-field continuation preserves source SQL collation instead of applying JavaScript order', async () => { + const rows = ['a', 'Z', 'é', '😀'].map(fieldName => ({ + created_at: new Date('2026-01-01T00:00:00.000Z'), + updated_at: new Date('2026-01-01T00:00:00.000Z'), + userId: 7, + certificateId: 1, + fieldName, + fieldValue: fieldName, + masterKey: 'key' + })) + const fixture = remoteReaderFixture(rows, 'certificateFields', 2) + const view = (await openRemoteSnapshot(fixture.transport))! + try { + let cursor: WalletSnapshotCursor | undefined + for (const row of rows) { + const page = await view.readPage('certificateFields', cursor, { maxRows: 1 }) + expect(page.rows).toEqual([row]) + expect(page.cursor!.after).toEqual([row.fieldName, 1]) + cursor = page.cursor + } + } finally { + await view.close() + } +}) + +test('cursor descriptors and numeric positions reject before invoking user accessors', async () => { + const { transport } = remoteReaderFixture([label(1), label(2)]) + const view = (await openRemoteSnapshot(transport))! + try { + const { cursor } = await view.readPage('txLabels', undefined, { maxRows: 1 }) + const getter = jest.fn(() => { + throw new Error('must not execute') + }) + const afterGetter = Object.defineProperty({ ...cursor! }, 'after', { get: getter }) + const hiddenAfter = Object.defineProperty({ ...cursor! }, 'after', { value: [1], enumerable: false }) + const missingAfter = { ...cursor!, renamedAfter: [1] } as Record + delete missingAfter.after + const arrayGetter = Object.defineProperty([1], '0', { get: getter }) + const sparse: unknown[] = [] + sparse.length = 1 + const invalid: unknown[] = [ + afterGetter, + hiddenAfter, + missingAfter, + { ...cursor, after: arrayGetter }, + { ...cursor, after: sparse }, + { ...cursor, after: null }, + { ...cursor, after: [] }, + { ...cursor, after: [1, 2] }, + { ...cursor, archivePosition: null }, + { ...cursor, archivePosition: [] }, + { ...cursor, archivePosition: { ...cursor!.archivePosition, version: 2 } } + ] + for (const value of [0, -1, 1.5, NaN, Infinity, Number.MAX_SAFE_INTEGER + 1]) { + invalid.push({ ...cursor, after: [value] }) + invalid.push({ ...cursor, archivePosition: { ...cursor!.archivePosition, rowOffset: value } }) + } + for (const value of [-1, 1.5, NaN, Infinity]) + invalid.push({ ...cursor, archivePosition: { ...cursor!.archivePosition, sequence: value } }) + for (const value of invalid) + await expect(view.readPage('txLabels', value as WalletSnapshotCursor)).rejects.toThrow('verified position') + expect(getter).not.toHaveBeenCalled() + expect((await view.readPage('txLabels', cursor)).rows).toEqual([label(2)]) + } finally { + await view.close() + } +}) + +test('page limits reject invalid numbers without consuming remote work', async () => { + const { transport, rpc } = remoteReaderFixture([label(1)]) + const view = (await openRemoteSnapshot(transport))! + try { + for (const field of ['maxRows', 'maxBytes'] as const) { + const ceiling = field === 'maxRows' ? 1000 : 16777216 + for (const value of [0, -1, 1.5, NaN, Infinity, ceiling + 1]) + await expect(view.readPage('txLabels', undefined, { [field]: value })).rejects.toThrow(field) + } + expect(rpc.mock.calls.filter(([method]) => method === 'readSnapshotArchivePage')).toHaveLength(0) + expect((await view.readPage('txLabels', undefined, { maxRows: 1000, maxBytes: 16777216 })).rows).toEqual([label(1)]) + } finally { + await view.close() + } +}) + +test('a delayed frame owns the only read slot through completion', async () => { + const { transport } = remoteReaderFixture([label(1)]) + const view = (await openRemoteSnapshot(transport))! + const original = transport.page.bind(transport) + let enter!: () => void + let release!: () => void + const entered = new Promise(resolve => { + enter = resolve + }) + const gate = new Promise(resolve => { + release = resolve + }) + jest.spyOn(transport, 'page').mockImplementation(async (...args) => { + enter() + await gate + return original(...args) + }) + const first = view.readPage('txLabels') + void first.catch(() => undefined) + try { + await entered + await expect(view.readPage('txLabels')).rejects.toThrow('already has a read in flight') + release() + expect((await first).rows).toEqual([label(1)]) + expect((await view.readPage('txLabels')).rows).toEqual([label(1)]) + } finally { + release() + await first.catch(() => undefined) + await view.close() + } +}) + +test('a read stops at conservative server expiry before the local rounded timer fires', async () => { + jest.useFakeTimers() + const monotonic = jest.spyOn(performance, 'now').mockReturnValue(0) + const { transport, rpc } = remoteReaderFixture([label(1)]) + const view = (await openRemoteSnapshot(transport, { lifetimeMs: 1000 }))! + try { + monotonic.mockReturnValue(999.1) + await expect(view.readPage('txLabels')).rejects.toThrow('Remote snapshot expired') + await view.closed + expect(rpc.mock.calls.filter(([method]) => method === 'readSnapshotArchivePage')).toHaveLength(0) + expect(rpc.mock.calls.filter(([method]) => method === 'cancelSnapshotArchiveRequest')).toHaveLength(1) + expect(jest.getTimerCount()).toBe(0) + } finally { + await view.close() + jest.useRealTimers() + } +}) + +test.each(['expiry', 'read'] as const)( + '%s failure preserves a separate failed cleanup acknowledgement', + async cause => { + jest.useFakeTimers() + const monotonic = jest.spyOn(performance, 'now').mockReturnValue(0) + const { transport } = remoteReaderFixture([label(1)]) + const view = (await openRemoteSnapshot(transport, { lifetimeMs: 1000 }))! + const cleanup = new Error('synthetic cancellation acknowledgement failure') + const original = new Error('synthetic authenticated frame failure') + const cancel = jest.spyOn(transport, 'cancelRequest').mockRejectedValue(cleanup) + try { + if (cause === 'expiry') monotonic.mockReturnValue(999.1) + else jest.spyOn(transport, 'page').mockRejectedValue(original) + const read = view.readPage('txLabels') + if (cause === 'expiry') await expect(read).rejects.toThrow('Remote snapshot expired') + else await expect(read).rejects.toBe(original) + await expect(view.closed).rejects.toBe(cleanup) + await expect(view.close()).rejects.toBe(cleanup) + expect(cancel).toHaveBeenCalledTimes(1) + expect(jest.getTimerCount()).toBe(0) + } finally { + await view.close().catch(() => undefined) + jest.useRealTimers() + } + } +) + +test('composite certificate cursors bind each key and accept the inclusive Unicode field-name limit', async () => { + const fieldName = '😀'.repeat(100) + const rows = [1, 2, 3].map(certificateId => ({ + created_at: new Date('2026-01-01T00:00:00.000Z'), + updated_at: new Date('2026-01-01T00:00:00.000Z'), + userId: 7, + certificateId, + fieldName, + fieldValue: 'value', + masterKey: 'synthetic key' + })) + const { transport, rpc } = remoteReaderFixture(rows, 'certificateFields', 2) + const view = (await openRemoteSnapshot(transport))! + try { + const first = await view.readPage('certificateFields', undefined, { maxRows: 1 }) + expect(first.rows).toEqual([rows[0]]) + expect(first.cursor!.after).toEqual([fieldName, 1]) + const reads = () => rpc.mock.calls.filter(([method]) => method === 'readSnapshotArchivePage').length + const initialReads = reads() + for (const after of [ + [fieldName + 'x', 1], + ['a'.repeat(101), 1], + [1, 1], + [fieldName, 0], + [fieldName, 1.5], + [fieldName, '1'] + ]) { + await expect(view.readPage('certificateFields', { ...first.cursor!, after })).rejects.toThrow('verified position') + expect(reads()).toBe(initialReads) + } + for (const after of [ + ['other', 1], + [fieldName, 2] + ]) + await expect(view.readPage('certificateFields', { ...first.cursor!, after })).rejects.toThrow('verified position') + const final = await view.readPage('certificateFields', first.cursor) + expect(final.rows).toEqual(rows.slice(1)) + expect(final.cursor!.after).toEqual([fieldName, 3]) + expect(final.cursor!.archivePosition).toEqual({ + version: 1, + archiveId: first.cursor!.archivePosition!.archiveId, + sequence: first.cursor!.archivePosition!.sequence + 1, + rowOffset: 1 + }) + expect(final.done).toBe(true) + } finally { + await view.close() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReaderHttp.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReaderHttp.test.ts new file mode 100644 index 000000000..3a7c22fc6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotReaderHttp.test.ts @@ -0,0 +1,444 @@ +import { StorageClient } from '../../remoting/StorageClient' +import { StorageClient as StorageMobile } from '../../remoting/StorageMobile' +import type { WalletStorageServerOptions } from '../../remoting/StorageServer' +import { snapshotHttpFixture, gate } from '../../../../test/utils/snapshotArchiveHttpFixtures' +import type { WalletReadSnapshot } from '../WalletReadSnapshot' +import { SnapshotCancelledError } from '../SnapshotCancelledError' +import { snapshotArchiveTables } from './SnapshotArchive' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { WalletStorageManager } from '../../WalletStorageManager' +import { KnexSnapshotArchiveRpc } from './KnexSnapshotArchiveRpc' + +afterEach(() => jest.restoreAllMocks()) + +// Opt this controlled HTTP fixture into the in-progress capability. Production +// advertisement stays disabled until the complete lifecycle qualification passes. +async function serveReader( + fixture: Awaited>, + options: Partial = {} +) { + const result = await fixture.serve(options) + const advertise = Reflect.get(result.server, 'snapshotArchiveSettings').bind(result.server) + jest.spyOn(result.server as never, 'snapshotArchiveSettings' as never).mockImplementation((async () => { + const settings = await advertise() + return settings.snapshotArchive === undefined ? settings : { ...settings, snapshotArchiveReaderVersion: 1 } + }) as never) + return result +} + +function rpcBody(init: RequestInit | undefined): { method: string; params: unknown[] } | undefined { + if (typeof init?.body !== 'string') return undefined + const body = JSON.parse(init.body) + return typeof body.method === 'string' ? body : undefined +} + +test.each([StorageClient, StorageMobile])( + 'authenticated %p cancellation waits for a second controller to confirm the original source cleanup', + async Client => { + const fixture = await snapshotHttpFixture() + const peer = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ ...fixture.storage.knex.client.config, pool: { min: 1, max: 1 } }) + }) + const replacement = new KnexSnapshotArchiveRpc(peer) + const reading = gate() + const allowRead = gate() + const destroying = gate() + const allowDestroy = gate() + const pendingCancellation = gate() + const outcomes: unknown[] = [] + let opening: Promise | undefined + try { + await peer.makeAvailable() + const { server, url } = await serveReader(fixture) + const originalRpc = Reflect.get(server, 'snapshotArchives') as KnexSnapshotArchiveRpc + const dispatch = originalRpc.dispatch.bind(originalRpc) + // Both clients use real authentication/framing through this HTTP edge. + // Its cancellation handler is routed to an independent SQL/controller instance. + jest.spyOn(originalRpc, 'dispatch').mockImplementation(async (method, params, identityKey) => { + if (method !== 'cancelSnapshotArchiveRequest') return await dispatch(method, params, identityKey) + const result = await replacement.dispatch(method, params, identityKey) + outcomes.push(result) + if (result !== true) pendingCancellation.resolve() + return result + }) + const originalOpen = fixture.storage.openSnapshotArchiveSource.bind(fixture.storage) + const open = jest.spyOn(fixture.storage, 'openSnapshotArchiveSource').mockImplementation(async (...args) => { + const source = (await originalOpen(...args))! + const owned = Reflect.get(fixture.storage, 'snapshotSyncSource') as StorageKnex + const destroy = owned.knex.client.destroyRawConnection.bind(owned.knex.client) + jest.spyOn(owned.knex.client, 'destroyRawConnection').mockImplementation(async connection => { + destroying.resolve() + await allowDestroy.promise + await destroy(connection) + }) + const read = source.readPage + source.readPage = async (table, cursor, limits) => { + reading.resolve() + await allowRead.promise + return await read(table, cursor, limits) + } + return source + }) + const foreignOpen = jest.spyOn(peer, 'openSnapshotArchiveSource') + const controller = new AbortController() + const client = new Client(fixture.wallet, url, { serverIdentityKey: fixture.serverIdentityKey }) + opening = client.getSnapshotSync()!.openSource(fixture.identityKey, { signal: controller.signal }) + let settled = false + void opening.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + await reading.promise + const request = await fixture.storage.knex('snapshot_archive_requests').first() + controller.abort() + await pendingCancellation.promise + expect(outcomes[0]).toEqual({ version: 1, outcome: 'cleanup-pending', requestId: request.requestId }) + expect(settled).toBe(false) + expect((await peer.knex('snapshot_archive_capacity').first()).archives).toBe(1) + allowRead.resolve() + await destroying.promise + expect(settled).toBe(false) + expect(await peer.knex('snapshot_archive_owners')).toHaveLength(1) + expect(await peer.knex('snapshot_archive_pages')).toHaveLength(0) + await peer.knex('tx_labels').where({ txLabelId: 1 }).update({ label: 'foreground while HTTP cleanup waits' }) + allowDestroy.resolve() + await expect(opening).rejects.toBeInstanceOf(SnapshotCancelledError) + expect(outcomes.at(-1)).toBe(true) + expect(open).toHaveBeenCalledTimes(1) + expect(foreignOpen).not.toHaveBeenCalled() + expect(await peer.knex('snapshot_archive_owners')).toHaveLength(0) + expect(await peer.knex('snapshot_archive_requests')).toHaveLength(0) + expect(await peer.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + } finally { + allowRead.resolve() + allowDestroy.resolve() + await opening?.catch(() => undefined) + await replacement.close() + await peer.destroy() + await fixture.close() + } + } +) + +test.each([StorageClient, StorageMobile])( + 'authenticated %p reader negotiates its first open and keeps the original archive through replacement', + async Client => { + const fixture = await snapshotHttpFixture() + let view: WalletReadSnapshot | undefined + try { + const { server, url } = await serveReader(fixture) + const client = new Client(fixture.wallet, url, { serverIdentityKey: fixture.serverIdentityKey }) + const source = client.getSnapshotSync()! + expect(client.isAvailable()).toBe(false) + const opening = jest.spyOn(fixture.storage, 'openSnapshotArchiveSource') + view = (await source.openSource(fixture.identityKey))! + expect(view.sourceStorage.storageIdentityKey).toBe('http-snapshot-source') + expect(view.user.identityKey).toBe(fixture.identityKey) + expect(view.user.created_at).toBeInstanceOf(Date) + expect(opening).toHaveBeenCalledTimes(1) + expect(Reflect.get(fixture.storage, 'snapshotSyncSource')).toBeUndefined() + const first = await view.readPage('txLabels', undefined, { maxRows: 1 }) + expect(first.done).toBe(false) + expect(first.cursor?.archivePosition).toMatchObject({ version: 1, rowOffset: 1 }) + const original = first.rows[0].label + first.rows[0].label = 'caller mutation' + await fixture.storage.knex('tx_labels').where({ txLabelId: 1 }).update({ label: 'foreground after capture' }) + expect((await view.readPage('txLabels', undefined, { maxRows: 1 })).rows[0].label).toBe(original) + const next = await view.readPage('txLabels', first.cursor, { maxRows: 1 }) + expect(next.rows).toHaveLength(1) + expect(next.rows[0].txLabelId).not.toBe(first.rows[0].txLabelId) + await server.close() + await serveReader(fixture, { port: Number(new URL(url).port) }) + const outputs = await view.readPage('outputs') + expect(outputs.rows.length).toBeGreaterThan(0) + expect(outputs.rows[0].lockingScript).toBeInstanceOf(Uint8Array) + expect(outputs.rows[0]).not.toHaveProperty('outputDescription') + for (const table of snapshotArchiveTables) { + const page = await view.readPage(table) + expect(page.done).toBe(true) + for (const row of page.rows) { + expect(row.created_at).toBeInstanceOf(Date) + expect(row.updated_at).toBeInstanceOf(Date) + if ('userId' in row) expect(row.userId).toBe(view.user.userId) + } + } + expect(opening).toHaveBeenCalledTimes(1) + await view.close() + await view.close() + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect(await fixture.storage.knex('snapshot_archive_pages')).toHaveLength(0) + expect(await fixture.storage.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 0, + reservedBytes: 0 + }) + } finally { + await view?.close().catch(() => undefined) + await fixture.close() + } + } +) + +test.each(['before', 'after'] as const)( + 'native fetch loss %s admission retries the identical tuple once through real authentication', + async lost => { + const fixture = await snapshotHttpFixture() + const nativeFetch = globalThis.fetch + const admissions: unknown[][] = [] + let view: WalletReadSnapshot | undefined + try { + const { url } = await serveReader(fixture) + jest.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const body = rpcBody(init) + if (body?.method === 'admitSnapshotArchive') { + admissions.push(body.params) + if (admissions.length === 1) { + if (lost === 'after') { + const response = await nativeFetch(input, init) + await response.body?.cancel() + } + throw new Error('synthetic native fetch connection loss') + } + } + return await nativeFetch(input, init) + }) + const client = new StorageClient(fixture.wallet, url) + const opening = jest.spyOn(fixture.storage, 'openSnapshotArchiveSource') + view = (await client.getSnapshotSync()!.openSource(fixture.identityKey))! + expect(view.isOpen).toBe(true) + expect(admissions).toHaveLength(2) + expect(admissions[0]).toEqual(admissions[1]) + expect(opening).toHaveBeenCalledTimes(1) + expect((await view.readPage('txLabels')).rows).toHaveLength(2) + await view.close() + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + } finally { + await view?.close().catch(() => undefined) + await fixture.close() + } + } +) + +test('an unauthenticated admission reply is not retried or treated as compatibility fallback', async () => { + const fixture = await snapshotHttpFixture() + const nativeFetch = globalThis.fetch + let admissions = 0 + try { + const { url } = await serveReader(fixture) + jest.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const response = await nativeFetch(input, init) + if (rpcBody(init)?.method !== 'admitSnapshotArchive') return response + admissions++ + const headers = new Headers(response.headers) + headers.delete('x-bsv-auth-signature') + return new Response(await response.arrayBuffer(), { status: response.status, headers }) + }) + const client = new StorageClient(fixture.wallet, url) + await expect(client.getSnapshotSync()!.openSource(fixture.identityKey)).rejects.toThrow('authentication') + expect(admissions).toBe(1) + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect(Reflect.get(fixture.storage, 'snapshotSyncSource')).toBeUndefined() + } finally { + await fixture.close() + } +}) + +test.each( + ['getSnapshotArchiveDirectory', 'readSnapshotArchivePage', 'cancelSnapshotArchiveRequest'].flatMap(method => + (['before', 'after'] as const).map(loss => ({ method, loss })) + ) +)('immutable $method recovers native loss $loss the request with one exact retry', async ({ method, loss }) => { + const fixture = await snapshotHttpFixture() + const nativeFetch = globalThis.fetch + const calls: unknown[][] = [] + let view: WalletReadSnapshot | undefined + try { + const { url } = await serveReader(fixture) + jest.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const body = rpcBody(init) + if (body?.method === method) { + calls.push(body.params) + if (calls.length === 1) { + if (loss === 'after') { + const response = await nativeFetch(input, init) + await response.body?.cancel() + } + throw new Error('synthetic native fetch connection loss') + } + } + return await nativeFetch(input, init) + }) + const opening = jest.spyOn(fixture.storage, 'openSnapshotArchiveSource') + const client = new StorageClient(fixture.wallet, url) + view = (await client.getSnapshotSync()!.openSource(fixture.identityKey))! + const expiry = view.expiresAt + expect((await view.readPage('txLabels')).rows).toHaveLength(2) + expect(view.expiresAt).toBe(expiry) + await view.close() + expect(calls).toHaveLength(2) + expect(calls[0]).toEqual(calls[1]) + expect(opening).toHaveBeenCalledTimes(1) + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect(await fixture.storage.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 0, + reservedBytes: 0 + }) + } finally { + await view?.close().catch(() => undefined) + await fixture.close() + } +}) + +test('a page authentication failure closes the view without retrying its content request', async () => { + const fixture = await snapshotHttpFixture() + const nativeFetch = globalThis.fetch + let pages = 0 + let view: WalletReadSnapshot | undefined + try { + const { url } = await serveReader(fixture) + jest.spyOn(globalThis, 'fetch').mockImplementation(async (input, init) => { + const response = await nativeFetch(input, init) + if (rpcBody(init)?.method !== 'readSnapshotArchivePage') return response + pages++ + const headers = new Headers(response.headers) + headers.delete('x-bsv-auth-signature') + return new Response(await response.arrayBuffer(), { status: response.status, headers }) + }) + view = (await new StorageClient(fixture.wallet, url).getSnapshotSync()!.openSource(fixture.identityKey))! + await expect(view.readPage('txLabels')).rejects.toThrow('authentication') + expect(pages).toBe(1) + expect(view.isOpen).toBe(false) + await view.closed + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + } finally { + await view?.close().catch(() => undefined) + await fixture.close() + } +}) + +test('cancellation during authenticated admission preserves its local type and drains the owned source', async () => { + const fixture = await snapshotHttpFixture() + const entered = gate() + const resume = gate() + const controller = new AbortController() + let pending: Promise | undefined + try { + const { url } = await serveReader(fixture) + const open = fixture.storage.openSnapshotArchiveSource.bind(fixture.storage) + jest.spyOn(fixture.storage, 'openSnapshotArchiveSource').mockImplementation(async (...args) => { + entered.resolve() + await resume.promise + return await open(...args) + }) + const client = new StorageMobile(fixture.wallet, url) + pending = client.getSnapshotSync()!.openSource(fixture.identityKey, { signal: controller.signal }) + void pending.catch(() => undefined) + await entered.promise + controller.abort() + resume.resolve() + await expect(pending).rejects.toBeInstanceOf(SnapshotCancelledError) + expect(Reflect.get(fixture.storage, 'snapshotSyncSource')).toBeUndefined() + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect(await fixture.storage.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 0, + reservedBytes: 0 + }) + } finally { + resume.resolve() + await pending?.then( + view => view?.close(), + () => undefined + ) + await fixture.close() + } +}) + +test.each([StorageClient, StorageMobile])( + 'authenticated %p sync commits archive positions and resumes cancellation into an occupied destination', + async Client => { + const fixture = await snapshotHttpFixture(true, false) + const directory = await mkdtemp(join(tmpdir(), 'snapshot-http-destination-')) + const destination = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + snapshotSync: true, + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) + try { + const { url } = await serveReader(fixture) + const sourceUser = (await fixture.storage.findUserByIdentityKey(fixture.identityKey))! + await fixture.storage.updateUser(sourceUser.userId, { activeStorage: 'http-snapshot-source' }) + for (let index = 0; index < 5; index++) { + await fixture.storage.insertTxLabel({ + txLabelId: 0, + userId: sourceUser.userId, + label: `label-${index}`, + isDeleted: index % 2 === 0, + created_at: new Date('2026-01-01T00:00:00.000Z'), + updated_at: new Date('2026-01-01T00:00:00.000Z') + }) + } + await destination.knex.raw('PRAGMA journal_mode = WAL') + await destination.migrate('HTTP destination', 'http-destination') + await destination.makeAvailable() + const { user: foreign } = await destination.findOrInsertUser(fixture.serverIdentityKey) + await destination.findOrInsertTxLabel(foreign.userId, 'foreign-existing') + const { user } = await destination.findOrInsertUser(fixture.identityKey) + await destination.updateUser(user.userId, { activeStorage: 'http-destination' }) + const manager = new WalletStorageManager(fixture.identityKey, destination) + await manager.makeAvailable() + const reader = new Client(fixture.wallet, url) + const legacy = jest.spyOn(destination, 'processSyncChunk') + const controller = new AbortController() + const partial = await manager.syncFromReaderResumable(fixture.identityKey, reader, { + maxItems: 2, + signal: controller.signal, + onProgress: progress => { + if (progress.snapshotCheckpoint?.cursor?.table === 'txLabels') controller.abort() + } + }) + expect(partial.status).toBe('cancelled') + expect(partial.snapshotCheckpoint?.cursor?.archivePosition).toMatchObject({ version: 1, rowOffset: 2 }) + expect(partial.snapshotCheckpoint).toEqual( + await destination.getSnapshotSync()!.checkpoint(fixture.identityKey, 'http-snapshot-source') + ) + expect(await destination.findTxLabels({ partial: { userId: user.userId } })).toHaveLength(2) + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + const complete = await manager.syncFromReaderResumable(fixture.identityKey, reader, { maxItems: 2 }) + expect(complete.status).toBe('completed') + expect(complete.snapshotCheckpoint?.done).toBe(true) + expect(complete.snapshotCheckpoint?.snapshotId).not.toBe(partial.snapshotCheckpoint?.snapshotId) + const labels = await destination.findTxLabels({ partial: { userId: user.userId } }) + expect(labels.map(row => [row.label, row.isDeleted]).sort()).toEqual( + Array.from({ length: 5 }, (_, index) => [`label-${index}`, index % 2 === 0]) + ) + expect((await destination.findTxLabels({ partial: { userId: foreign.userId } })).map(row => row.label)).toEqual([ + 'foreign-existing' + ]) + expect((await destination.findUserByIdentityKey(fixture.identityKey))!.activeStorage).toBe('http-destination') + expect(legacy).not.toHaveBeenCalled() + expect(await fixture.storage.knex('snapshot_archive_requests')).toHaveLength(0) + expect(await fixture.storage.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 0, + reservedBytes: 0 + }) + } finally { + await destination.destroy() + await fixture.close() + await rm(directory, { recursive: true, force: true }) + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotRows.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotRows.test.ts new file mode 100644 index 000000000..0dc46acd4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotRows.test.ts @@ -0,0 +1,340 @@ +import type { PackedSnapshotRow, WalletSnapshotTables } from '../WalletReadSnapshot' +import { encodeSyncTransfer } from '../../remoting/SyncTransfer' +import { decodeRemoteSnapshotFrame, detachRemoteSnapshotRow, remoteSnapshotKeys } from './RemoteSnapshotRows' +import { snapshotArchiveLimits } from './SnapshotArchive' +import type { SnapshotArchiveReceipt } from './SnapshotArchiveDirectory' +import { label } from '../../../../test/utils/remoteSnapshotReaderFixtures' + +const receipt: SnapshotArchiveReceipt = { sequence: 0, table: 'txLabels', rows: 1, done: true, digest: 'a'.repeat(64) } +const encode = (rows: unknown[], changes: object = {}) => + encodeSyncTransfer({ version: 1, table: 'txLabels', rows, ...changes }) + +test('a raw row frame reconstructs fresh dates, retains exact Unicode and charges its payload budget', () => { + const row = label(1, 'é\u0000😀') + const decoded = decodeRemoteSnapshotFrame(encode([row]), receipt, 7) + expect(decoded.rows).toEqual([row]) + expect(decoded.charges).toEqual([6 * 64 + row.label.length * 2]) + const first = detachRemoteSnapshotRow(decoded.rows[0]) + const second = detachRemoteSnapshotRow(decoded.rows[0]) + expect(first).toEqual(row) + expect(second).toEqual(row) + expect(first.created_at).not.toBe(second.created_at) + ;(first.created_at as Date).setTime(0) + first.label = 'caller mutation' + expect(detachRemoteSnapshotRow(decoded.rows[0])).toEqual(row) +}) + +test('transaction bytes remain packed and detached, including the optional no-send recovery field', () => { + const row = { + created_at: new Date('2026-01-01T00:00:00.000Z'), + updated_at: new Date('2026-01-02T00:00:00.000Z'), + transactionId: 3, + userId: 7, + status: 'completed', + reference: 'reference', + isOutgoing: true, + satoshis: 1, + description: 'synthetic transaction', + rawTx: Uint8Array.of(0, 255), + noSendExpiryReclaimRawTx: Uint8Array.of(1, 128, 255) + } + const result = decodeRemoteSnapshotFrame( + encodeSyncTransfer({ version: 1, table: 'transactions', rows: [row] }), + { ...receipt, table: 'transactions' }, + 7 + ) + expect(result.rows).toEqual([row]) + expect(result.charges).toEqual([ + Object.values(row).reduce( + (sum, value) => + sum + 64 + (typeof value === 'string' ? value.length * 2 : value instanceof Uint8Array ? value.length * 2 : 0), + 0 + ) + ]) + const first = detachRemoteSnapshotRow(result.rows[0]) + ;(first.rawTx as Uint8Array).fill(42) + ;(first.noSendExpiryReclaimRawTx as Uint8Array).fill(42) + expect(detachRemoteSnapshotRow(result.rows[0])).toEqual(row) + expect(Array.isArray(result.rows[0].rawTx)).toBe(false) +}) + +test('unknown columns, foreign profiles, missing required fields and wrong cell representations reject', () => { + const invalid = [ + { ...label(1), userId: 8 }, + { ...label(1), userId: 0 }, + { ...label(1), txLabelId: 1.5 }, + { ...label(1), isDeleted: 1 }, + { ...label(1), label: 1 }, + { ...label(1), extra: 'column' }, + { ...label(1), updated_at: '2026-01-01' }, + { ...label(1), created_at: 'not-a-date' }, + { ...label(1), created_at: 0 }, + { ...label(1), label: null }, + Object.fromEntries(Object.entries(label(1)).filter(([name]) => name !== 'isDeleted')), + Object.assign(label(1), Object.fromEntries(Array.from({ length: 65 }, (_, n) => [`extra${n}`, n]))) + ] + for (const row of invalid) + expect(() => decodeRemoteSnapshotFrame(encode([row]), receipt, 7)).toThrow('Invalid snapshot archive row frame') +}) + +test('a frame must match its authenticated table and exact row count within hard byte and row bounds', () => { + for (const bytes of [new Uint8Array(0), new Uint8Array(snapshotArchiveLimits.pageBytes + 1)]) { + expect(() => decodeRemoteSnapshotFrame(bytes, receipt, 7)).toThrow('Invalid snapshot archive row frame') + } + for (const fields of [{ version: 2 }, { table: 'outputs' }, { extra: true }, { rows: {} }]) { + expect(() => decodeRemoteSnapshotFrame(encode([label(1)], fields), receipt, 7)).toThrow( + 'Invalid snapshot archive row frame' + ) + } + expect(() => decodeRemoteSnapshotFrame(encode([]), receipt, 7)).toThrow('Invalid snapshot archive row frame') + const large = Array.from({ length: snapshotArchiveLimits.rowsPerPage + 1 }, (_, n) => label(n + 1)) + expect(() => decodeRemoteSnapshotFrame(encode(large), { ...receipt, rows: large.length }, 7)).toThrow( + 'Invalid snapshot archive row frame' + ) + expect(decodeRemoteSnapshotFrame(encode([]), { ...receipt, rows: 0 }, 7)).toEqual({ rows: [], charges: [] }) +}) + +test('source-key declarations are detached and inherited table names never select a schema', () => { + const keys = remoteSnapshotKeys('certificateFields') as string[] + expect(keys).toEqual(['fieldName', 'certificateId']) + keys[0] = 'caller replacement' + expect(remoteSnapshotKeys('certificateFields')).toEqual(['fieldName', 'certificateId']) + for (const name of ['toString', '__proto__', 'constructor', 'unknown']) { + expect(() => remoteSnapshotKeys(name as 'txLabels')).toThrow('Invalid snapshot archive row frame') + } +}) + +test.each([null, 7, 'row', [], Uint8Array.of(1)])('non-record row %p cannot be returned as wallet data', input => { + expect(() => decodeRemoteSnapshotFrame(encode([input]), receipt, 7)).toThrow('Invalid snapshot archive row frame') +}) + +test('binary, numeric and certificate-field values retain strict representations and inclusive Unicode bounds', () => { + const when = new Date('2026-01-01T00:00:00.000Z') + const proven = { + created_at: when, + updated_at: when, + provenTxId: 1, + txid: 'transaction', + height: 0, + index: 0, + merklePath: new Uint8Array(), + rawTx: new Uint8Array(), + blockHash: 'block', + merkleRoot: 'root' + } + const decode = (table: 'provenTxs' | 'certificateFields', input: object) => + decodeRemoteSnapshotFrame(encodeSyncTransfer({ version: 1, table, rows: [input] }), { ...receipt, table }, 7) + expect(decode('provenTxs', proven).rows).toEqual([proven]) + for (const value of [null, [], [0, 255], 'binary']) + expect(() => decode('provenTxs', { ...proven, rawTx: value })).toThrow('Invalid snapshot archive row frame') + for (const value of [null, '0', true]) + expect(() => decode('provenTxs', { ...proven, height: value })).toThrow('Invalid snapshot archive row frame') + const field = { + created_at: when, + updated_at: when, + userId: 7, + certificateId: 1, + fieldName: '😀'.repeat(100), + fieldValue: 'value', + masterKey: 'key' + } + expect(decode('certificateFields', field).rows).toEqual([field]) + for (const fieldName of [1, 'a'.repeat(101), '😀'.repeat(101)]) + expect(() => decode('certificateFields', { ...field, fieldName })).toThrow('Invalid snapshot archive row frame') +}) + +test('all thirteen public row shapes retain their columns and reject changed representations and profiles', () => { + const when = new Date('2026-01-01T00:00:00.000Z') + const timestamps = { created_at: when, updated_at: when } + const txid = '12'.repeat(32) + const identity = '02' + '34'.repeat(32) + const packed = Uint8Array.of(0, 128, 255) + type Complete = Required> + const examples: { [T in keyof WalletSnapshotTables]: Complete } = { + provenTxs: { + ...timestamps, + provenTxId: 1, + txid, + height: 0, + index: 0, + merklePath: packed, + rawTx: packed, + blockHash: txid, + merkleRoot: txid + }, + outputBaskets: { + ...timestamps, + basketId: 1, + userId: 7, + name: 'default', + numberOfDesiredUTXOs: 10, + minimumDesiredUTXOValue: 1000, + isDeleted: false + }, + commissions: { + ...timestamps, + commissionId: 1, + userId: 7, + transactionId: 1, + satoshis: 1, + keyOffset: 'synthetic offset', + isRedeemed: false, + lockingScript: packed + }, + outputTags: { ...timestamps, outputTagId: 1, userId: 7, tag: 'tag', isDeleted: false }, + outputTagMaps: { ...timestamps, outputTagId: 1, outputId: 2, isDeleted: false }, + txLabels: label(1), + txLabelMaps: { ...timestamps, txLabelId: 1, transactionId: 2, isDeleted: false }, + certificateFields: { + ...timestamps, + userId: 7, + certificateId: 1, + fieldName: 'name', + fieldValue: 'value', + masterKey: 'synthetic key' + }, + transactions: { + ...timestamps, + transactionId: 1, + userId: 7, + provenTxId: 2, + status: 'completed', + reference: 'cmVm', + isOutgoing: true, + satoshis: 1000, + description: 'synthetic history', + version: 2, + lockTime: 0, + txid, + inputBEEF: packed, + rawTx: packed, + noSendExpiryMode: 'seconds', + noSendExpiryValue: 60, + noSendExpiryDeadline: 1790812860, + noSendExpiryState: 'reclaimed', + noSendExpiryAnchorTxid: txid, + noSendExpiryAnchorVout: 1, + noSendExpiryReleasedAt: 1790812861, + noSendExpiryObservedAt: 1790812862, + noSendExpiryReclaimTxid: txid, + noSendExpiryReclaimRawTx: packed, + noSendExpiryReclaimDerivationPrefix: 'cHJlZml4', + noSendExpiryReclaimDerivationSuffix: 'c3VmZml4', + noSendExpiryReclaimSatoshis: 500 + }, + outputs: { + ...timestamps, + outputId: 1, + userId: 7, + transactionId: 1, + basketId: 1, + spendable: false, + change: false, + outputDescription: 'synthetic output', + vout: 0, + satoshis: 1000, + providedBy: 'you-and-storage', + purpose: 'purpose', + type: 'P2PKH', + txid, + senderIdentityKey: identity, + derivationPrefix: 'cHJlZml4', + derivationSuffix: 'c3VmZml4', + customInstructions: 'retained metadata', + spentBy: 2, + sequenceNumber: 0, + spendingDescription: 'spent description', + scriptLength: 3, + scriptOffset: 0, + lockingScript: packed + }, + provenTxReqs: { + ...timestamps, + provenTxReqId: 1, + provenTxId: 2, + status: 'completed', + attempts: 0, + notified: true, + txid, + batch: 'batch', + history: '{}', + notify: '{}', + rawTx: packed, + inputBEEF: packed, + wasBroadcast: false, + rebroadcastAttempts: 0 + }, + certificates: { + ...timestamps, + certificateId: 1, + userId: 7, + type: 'dHlwZQ==', + serialNumber: 'c2VyaWFs', + certifier: identity, + subject: identity, + verifier: identity, + revocationOutpoint: txid + '.0', + signature: '1234', + isDeleted: false + }, + syncStates: { + ...timestamps, + syncStateId: 1, + userId: 7, + storageIdentityKey: identity, + storageName: 'original storage', + status: 'success', + init: true, + refNum: 'reference', + syncMap: '{}', + when, + satoshis: 1000, + errorLocal: 'historical local error', + errorOther: 'historical remote error' + } + } + for (const [name, example] of Object.entries(examples)) { + const table = name as keyof typeof examples + const decode = (row: object) => + decodeRemoteSnapshotFrame(encodeSyncTransfer({ version: 1, table, rows: [row] }), { ...receipt, table }, 7) + expect(decode(example).rows).toEqual([example]) + if ('userId' in example) + expect(() => decode({ ...example, userId: 8 })).toThrow('Invalid snapshot archive row frame') + for (const [field, value] of Object.entries(example)) { + // Derive only the invalid representation from independently typed public + // fixtures, without reading the decoder's private schema declaration. + const wrong = typeof value === 'string' ? 1 : typeof value === 'boolean' ? 0 : 'wrong representation' + expect(() => decode({ ...example, [field]: wrong })).toThrow('Invalid snapshot archive row frame') + } + } +}) + +test('every source table preserves the established scalar or composite key order', () => { + const expected: Record = { + provenTxs: ['provenTxId'], + provenTxReqs: ['provenTxReqId'], + outputBaskets: ['basketId'], + transactions: ['transactionId'], + commissions: ['commissionId'], + outputs: ['outputId'], + outputTags: ['outputTagId'], + outputTagMaps: ['outputTagId', 'outputId'], + txLabels: ['txLabelId'], + txLabelMaps: ['txLabelId', 'transactionId'], + certificates: ['certificateId'], + certificateFields: ['fieldName', 'certificateId'], + syncStates: ['syncStateId'] + } + for (const [name, keys] of Object.entries(expected)) { + const table = name as keyof WalletSnapshotTables + const actual = remoteSnapshotKeys(table) as string[] + expect(actual).toEqual(keys) + actual.length = 0 + expect(remoteSnapshotKeys(table)).toEqual(keys) + } +}) + +test('the inclusive authenticated row limit accepts a full page', () => { + const rows = Array.from({ length: snapshotArchiveLimits.rowsPerPage }, (_, index) => label(index + 1)) + expect(decodeRemoteSnapshotFrame(encode(rows), { ...receipt, rows: rows.length }, 7).rows).toEqual(rows) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotRows.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotRows.ts new file mode 100644 index 000000000..4a7db0778 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/RemoteSnapshotRows.ts @@ -0,0 +1,331 @@ +import { decodeSyncTransfer } from '../../remoting/SyncTransfer' +import type { WalletSnapshotTable } from '../WalletReadSnapshot' +import { snapshotArchiveLimits } from './SnapshotArchive' +import type { SnapshotArchiveReceipt } from './SnapshotArchiveDirectory' + +type Cell = string | number | boolean | Date | Uint8Array +type Kind = 'string' | 'number' | 'boolean' | 'date' | 'bytes' | 'id' | 'fieldName' +export type RemoteSnapshotRow = Readonly> +interface Schema { + keys: readonly string[] + owned: boolean + required: Readonly> + optional?: Readonly> +} +const timestamps = { created_at: 'date', updated_at: 'date' } as const + +// Version-one raw SQL row shapes. This validates representation and ownership; +// portable semantics and proof authentication remain independent consumers. +const schemas: Record = { + provenTxs: { + keys: ['provenTxId'], + owned: false, + required: { + ...timestamps, + provenTxId: 'id', + txid: 'string', + height: 'number', + index: 'number', + merklePath: 'bytes', + rawTx: 'bytes', + blockHash: 'string', + merkleRoot: 'string' + } + }, + provenTxReqs: { + keys: ['provenTxReqId'], + owned: false, + required: { + ...timestamps, + provenTxReqId: 'id', + status: 'string', + attempts: 'number', + notified: 'boolean', + txid: 'string', + history: 'string', + notify: 'string', + rawTx: 'bytes' + }, + optional: { + provenTxId: 'id', + batch: 'string', + inputBEEF: 'bytes', + wasBroadcast: 'boolean', + rebroadcastAttempts: 'number' + } + }, + outputBaskets: { + keys: ['basketId'], + owned: true, + required: { + ...timestamps, + basketId: 'id', + userId: 'id', + name: 'string', + numberOfDesiredUTXOs: 'number', + minimumDesiredUTXOValue: 'number', + isDeleted: 'boolean' + } + }, + transactions: { + keys: ['transactionId'], + owned: true, + required: { + ...timestamps, + transactionId: 'id', + userId: 'id', + status: 'string', + reference: 'string', + isOutgoing: 'boolean', + satoshis: 'number', + description: 'string' + }, + optional: { + provenTxId: 'id', + version: 'number', + lockTime: 'number', + txid: 'string', + inputBEEF: 'bytes', + rawTx: 'bytes', + noSendExpiryMode: 'string', + noSendExpiryValue: 'number', + noSendExpiryDeadline: 'number', + noSendExpiryState: 'string', + noSendExpiryAnchorTxid: 'string', + noSendExpiryAnchorVout: 'number', + noSendExpiryReleasedAt: 'number', + noSendExpiryObservedAt: 'number', + noSendExpiryReclaimTxid: 'string', + noSendExpiryReclaimRawTx: 'bytes', + noSendExpiryReclaimDerivationPrefix: 'string', + noSendExpiryReclaimDerivationSuffix: 'string', + noSendExpiryReclaimSatoshis: 'number' + } + }, + commissions: { + keys: ['commissionId'], + owned: true, + required: { + ...timestamps, + commissionId: 'id', + userId: 'id', + transactionId: 'id', + satoshis: 'number', + keyOffset: 'string', + isRedeemed: 'boolean', + lockingScript: 'bytes' + } + }, + outputs: { + keys: ['outputId'], + owned: true, + required: { + ...timestamps, + outputId: 'id', + userId: 'id', + transactionId: 'id', + spendable: 'boolean', + change: 'boolean', + vout: 'number', + satoshis: 'number', + providedBy: 'string', + purpose: 'string', + type: 'string' + }, + optional: { + // SQL permits this legacy column to be null; normalized source rows omit it. + outputDescription: 'string', + basketId: 'id', + txid: 'string', + senderIdentityKey: 'string', + derivationPrefix: 'string', + derivationSuffix: 'string', + customInstructions: 'string', + spentBy: 'id', + sequenceNumber: 'number', + spendingDescription: 'string', + scriptLength: 'number', + scriptOffset: 'number', + lockingScript: 'bytes' + } + }, + outputTags: { + keys: ['outputTagId'], + owned: true, + required: { ...timestamps, outputTagId: 'id', userId: 'id', tag: 'string', isDeleted: 'boolean' } + }, + outputTagMaps: { + keys: ['outputTagId', 'outputId'], + owned: false, + required: { ...timestamps, outputTagId: 'id', outputId: 'id', isDeleted: 'boolean' } + }, + txLabels: { + keys: ['txLabelId'], + owned: true, + required: { ...timestamps, txLabelId: 'id', userId: 'id', label: 'string', isDeleted: 'boolean' } + }, + txLabelMaps: { + keys: ['txLabelId', 'transactionId'], + owned: false, + required: { ...timestamps, txLabelId: 'id', transactionId: 'id', isDeleted: 'boolean' } + }, + certificates: { + keys: ['certificateId'], + owned: true, + required: { + ...timestamps, + certificateId: 'id', + userId: 'id', + type: 'string', + serialNumber: 'string', + certifier: 'string', + subject: 'string', + revocationOutpoint: 'string', + signature: 'string', + isDeleted: 'boolean' + }, + optional: { verifier: 'string' } + }, + certificateFields: { + keys: ['fieldName', 'certificateId'], + owned: true, + required: { + ...timestamps, + userId: 'id', + certificateId: 'id', + fieldName: 'fieldName', + fieldValue: 'string', + masterKey: 'string' + } + }, + syncStates: { + keys: ['syncStateId'], + owned: true, + required: { + ...timestamps, + syncStateId: 'id', + userId: 'id', + storageIdentityKey: 'string', + storageName: 'string', + status: 'string', + init: 'boolean', + refNum: 'string', + syncMap: 'string' + }, + optional: { when: 'date', satoshis: 'number', errorLocal: 'string', errorOther: 'string' } + } +} + +function invalid(): never { + throw new TypeError('Invalid snapshot archive row frame') +} + +export function remoteSnapshotKeys(table: WalletSnapshotTable): readonly string[] { + if (!Object.hasOwn(schemas, table)) invalid() + return [...schemas[table].keys] +} + +function record(value: unknown): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) invalid() + // This private helper only receives decoded bytes. JSON parsing and binary + // restoration create enumerable string-keyed data properties, never accessors + // or symbols; arbitrary caller objects are validated at the cursor boundary. + if (Object.keys(value).length > 64) invalid() + return Object.fromEntries(Object.entries(value)) +} + +function dateCell(value: unknown): Date { + if (typeof value !== 'string' || value.length > 32) invalid() + const date = new Date(value) + if (!Number.isFinite(date.getTime()) || date.toISOString() !== value) invalid() + return date +} + +function cell(value: unknown, kind: Kind): Cell { + switch (kind) { + case 'bytes': + if (!(value instanceof Uint8Array)) invalid() + return value + case 'date': + return dateCell(value) + case 'id': + if (!Number.isSafeInteger(value) || (value as number) < 1) invalid() + return value as number + case 'fieldName': + if (typeof value !== 'string' || value.length > 200 || Array.from(value).length > 100) invalid() + return value + case 'number': + if (typeof value !== 'number' || !Number.isFinite(value)) invalid() + return value + case 'boolean': + if (typeof value !== 'boolean') invalid() + return value + case 'string': + if (typeof value !== 'string') invalid() + return value + } +} + +function fieldKind(schema: Schema, name: string): Kind | undefined { + if (Object.hasOwn(schema.required, name)) return schema.required[name] + if (schema.optional !== undefined && Object.hasOwn(schema.optional, name)) return schema.optional[name] + return undefined +} + +function row(input: unknown, schema: Schema, userId: number): RemoteSnapshotRow { + const fields = record(input) + for (const name of Object.keys(schema.required)) if (!Object.hasOwn(fields, name)) invalid() + const result: Record = {} + for (const [name, value] of Object.entries(fields)) { + const kind = fieldKind(schema, name) + if (kind === undefined) invalid() + Object.defineProperty(result, name, { value: cell(value, kind), enumerable: true }) + } + if (schema.owned && result.userId !== userId) invalid() + return result +} + +export interface RemoteSnapshotFrame { + readonly rows: readonly RemoteSnapshotRow[] + readonly charges: readonly number[] +} + +function cellCharge(entry: Cell): number { + if (typeof entry === 'string') return 64 + entry.length * 2 + if (entry instanceof Uint8Array) return 64 + entry.byteLength * 2 + return 64 +} + +/** Decode one independently bounded frame only after its receipt hash was verified. */ +export function decodeRemoteSnapshotFrame( + bytes: Uint8Array, + receipt: SnapshotArchiveReceipt, + userId: number +): RemoteSnapshotFrame { + if (bytes.length < 1 || bytes.length > snapshotArchiveLimits.pageBytes) invalid() + remoteSnapshotKeys(receipt.table) + const frame = record(decodeSyncTransfer(bytes)) + if ( + Object.keys(frame).length !== 3 || + frame.version !== 1 || + frame.table !== receipt.table || + !Array.isArray(frame.rows) || + frame.rows.length !== receipt.rows || + frame.rows.length > snapshotArchiveLimits.rowsPerPage + ) + invalid() + const rows = frame.rows.map(input => row(input, schemas[receipt.table], userId)) + const charges = rows.map(value => Object.values(value).reduce((sum, entry) => sum + cellCharge(entry), 0)) + return { rows, charges } +} + +function detachCell(entry: Cell): Cell { + if (entry instanceof Uint8Array) return new Uint8Array(entry) + if (entry instanceof Date) return new Date(entry) + return entry +} + +/** The cached frame remains private; callers own every returned mutable value. */ +export function detachRemoteSnapshotRow(value: RemoteSnapshotRow): Record { + return Object.fromEntries(Object.entries(value).map(([name, entry]) => [name, detachCell(entry)])) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchive.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchive.ts new file mode 100644 index 000000000..10ab5c7c5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchive.ts @@ -0,0 +1,66 @@ +import type { TableSettings, TableUser } from '../../schema/tables' +import type { WalletSnapshotTable } from '../WalletReadSnapshot' + +/** Persistent transport staging; these auxiliary records never enter BRC-38. */ +export const snapshotArchiveTables = Object.freeze([ + 'provenTxs', + 'provenTxReqs', + 'outputBaskets', + 'transactions', + 'commissions', + 'outputs', + 'outputTags', + 'outputTagMaps', + 'txLabels', + 'txLabelMaps', + 'certificates', + 'certificateFields', + 'syncStates' +] as const satisfies readonly WalletSnapshotTable[]) + +export const snapshotArchiveLimits = Object.freeze({ + archives: 8, + totalBytes: 128 * 1024 * 1024, + archiveBytes: 32 * 1024 * 1024, + pageBytes: 1024 * 1024, + pages: 4096, + rowsPerPage: 1000, + lifetimeMs: 3600000, + bindingBytes: 65536, + headerCharge: 4096, + pageCharge: 512 +}) + +export interface SnapshotArchiveBinding { + version: 1 + /** The original retained source view, never a replacement replica checkpoint. */ + snapshotId: string + sourceStorage: TableSettings + sourceSchema: string + user: TableUser +} + +export interface SnapshotArchiveWriter { + archiveId: string + /** Internal capture ownership only. Never return this token to an RPC caller. */ + writerToken: string +} + +export interface SnapshotArchiveManifest { + version: 1 + archiveId: string + binding: SnapshotArchiveBinding + expiresAt: number + pages: number + rows: number + digest: string +} + +export interface SnapshotArchivePage { + sequence: number + table: WalletSnapshotTable + rows: number + done: boolean + bytes: Uint8Array + digest: string +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveAdmission.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveAdmission.test.ts new file mode 100644 index 000000000..89568276a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveAdmission.test.ts @@ -0,0 +1,77 @@ +import { validateSnapshotArchiveAdmission, SnapshotArchiveAdmissionLimitError } from './SnapshotArchiveAdmission' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' + +const fields = { version: 1 as const, nonce: 'a'.repeat(64), notAfter: 1790812800000, maxBytes: 32768 } +const request = { ...fields, requestId: snapshotArchiveRequestId(fields) } +const receipt = { version: 1, requestId: request.requestId, expiresAt: request.notAfter, state: 'building' } +const refused = { version: 1, outcome: 'resource-limited', requestId: request.requestId, expiresAt: request.notAfter } + +test('accepted and transient refused outcomes are distinct, detached and immutable', () => { + const accepted = validateSnapshotArchiveAdmission({ version: 1, outcome: 'accepted', receipt }, request) + expect(accepted).toEqual({ version: 1, outcome: 'accepted', receipt }) + expect(Object.isFrozen(accepted)).toBe(true) + if (accepted.outcome !== 'accepted') throw new Error('Expected accepted fixture') + expect(accepted.receipt).not.toBe(receipt) + expect(Object.isFrozen(accepted.receipt)).toBe(true) + const result = validateSnapshotArchiveAdmission(refused, request) + expect(result).toEqual(refused) + expect(result).not.toBe(refused) + expect(Object.isFrozen(result)).toBe(true) + expect(result).not.toHaveProperty('receipt') + expect(new SnapshotArchiveAdmissionLimitError('occupied')).toBeInstanceOf(SnapshotResourceLimitError) +}) + +test.each([null, undefined, [], true, 1, 'accepted'])('rejects non-record admission %p', value => { + expect(() => validateSnapshotArchiveAdmission(value, request)).toThrow('Invalid snapshot archive admission') +}) + +test.each([ + { version: 2 }, + { outcome: 'ready' }, + { requestId: 'b'.repeat(64) }, + { expiresAt: request.notAfter + 1 }, + { receipt }, + { claimToken: 'unwanted' }, + { [Symbol('extra')]: true } +])('refusal binds exactly to the original request and field set %p', fields => { + expect(() => validateSnapshotArchiveAdmission({ ...refused, ...fields }, request)).toThrow( + 'Invalid snapshot archive admission' + ) +}) + +test('own enumerable data fields are required without invoking getters', () => { + for (const name of Object.keys(refused)) { + const missing = { ...refused } as Record + delete missing[name] + expect(() => validateSnapshotArchiveAdmission(missing, request)).toThrow( + 'Invalid snapshot archive admission outcome' + ) + Object.defineProperty(missing, name, { value: Reflect.get(refused, name), configurable: true }) + expect(() => validateSnapshotArchiveAdmission(missing, request)).toThrow( + 'Invalid snapshot archive admission outcome' + ) + Object.defineProperty(missing, name, { + get: () => { + throw new Error('Getter must not run') + }, + enumerable: true + }) + expect(() => validateSnapshotArchiveAdmission(missing, request)).toThrow('Invalid snapshot archive admission') + } + expect(() => validateSnapshotArchiveAdmission(Object.create(refused), request)).toThrow( + 'Invalid snapshot archive admission outcome' + ) + expect(() => + validateSnapshotArchiveAdmission( + { version: 1, outcome: 'accepted', receipt: { ...receipt, expiresAt: 1 } }, + request + ) + ).toThrow('Invalid snapshot archive protocol') +}) + +test.each([null, undefined, true, 1, 'request', {}])('an admission cannot bind to invalid request %p', value => { + expect(() => validateSnapshotArchiveAdmission(refused, value as unknown as typeof request)).toThrow( + 'Invalid snapshot archive creation request' + ) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveAdmission.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveAdmission.ts new file mode 100644 index 000000000..4c1afc5a1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveAdmission.ts @@ -0,0 +1,60 @@ +import { parseSnapshotArchiveReaderRequest, type SnapshotArchiveReaderRequest } from './SnapshotArchiveReaderRequest' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { + parseSnapshotArchiveRequest, + type SnapshotArchiveRequest, + type SnapshotArchiveRequestReceipt +} from './SnapshotArchiveRequest' +import { validateSnapshotArchiveRequestReceipt } from './SnapshotArchiveProtocol' + +/** Only pre-admission capacity refusals carry this type; cleanup errors do not. */ +export class SnapshotArchiveAdmissionLimitError extends SnapshotResourceLimitError {} + +export type SnapshotArchiveAdmission = + | { version: 1; outcome: 'accepted'; receipt: Readonly } + | { version: 1; outcome: 'resource-limited'; requestId: string; expiresAt: number } + +function invalid(): never { + throw new TypeError('Invalid snapshot archive admission outcome') +} + +/** A refusal is transient and never stands in for an immutable durable receipt. */ +export function validateSnapshotArchiveAdmission( + input: unknown, + value: SnapshotArchiveRequest | SnapshotArchiveReaderRequest +): Readonly { + const version = + value !== null && typeof value === 'object' ? Object.getOwnPropertyDescriptor(value, 'version')?.value : undefined + const request = version === 2 ? parseSnapshotArchiveReaderRequest(value) : parseSnapshotArchiveRequest(value) + if (input === null || typeof input !== 'object' || Array.isArray(input)) invalid() + const property = Object.getOwnPropertyDescriptor(input, 'outcome') + if (property === undefined || !('value' in property)) invalid() + const names = + property.value === 'accepted' ? ['version', 'outcome', 'receipt'] : ['version', 'outcome', 'requestId', 'expiresAt'] + if (Reflect.ownKeys(input).length !== names.length) invalid() + const fields: Record = {} + for (const name of names) { + const item = Object.getOwnPropertyDescriptor(input, name) + if (item === undefined || !('value' in item) || !item.enumerable) invalid() + fields[name] = item.value + } + if (fields.version !== 1) invalid() + if (fields.outcome === 'accepted') + return Object.freeze({ + version: 1, + outcome: 'accepted', + receipt: validateSnapshotArchiveRequestReceipt(fields.receipt, request) + }) + if ( + fields.outcome !== 'resource-limited' || + fields.requestId !== request.requestId || + fields.expiresAt !== request.notAfter + ) + invalid() + return Object.freeze({ + version: 1, + outcome: 'resource-limited', + requestId: request.requestId, + expiresAt: request.notAfter + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveCleanup.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveCleanup.ts new file mode 100644 index 000000000..0a0ec22fc --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveCleanup.ts @@ -0,0 +1,84 @@ +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { retrySnapshotArchiveOperation } from './SnapshotArchiveTransportFailure' + +/** Fencing has succeeded, but the source has not acknowledged cleanup. */ +export class SnapshotArchiveCleanupPendingError extends WERR_INVALID_OPERATION { + constructor() { + super('Snapshot archive source cleanup is pending') + } +} + +export interface SnapshotArchiveCleanupPending { + version: 1 + outcome: 'cleanup-pending' + requestId: string +} + +/** The existing true acknowledgement remains the only successful completion. */ +export function validateSnapshotArchiveCancellation(input: unknown, requestId: string): void { + if (input === true) return + if (input === null || typeof input !== 'object' || Array.isArray(input) || Reflect.ownKeys(input).length !== 3) + throw new TypeError('Invalid snapshot archive cancellation receipt') + const expected = { version: 1, outcome: 'cleanup-pending', requestId } + for (const [key, value] of Object.entries(expected)) { + const field = Object.getOwnPropertyDescriptor(input, key) + if (field === undefined || !('value' in field) || !field.enumerable || field.value !== value) + throw new TypeError('Invalid snapshot archive cancellation receipt') + } + throw new SnapshotArchiveCleanupPendingError() +} + +/** Separate fixed cleanup allowance; neither the source lease nor request identity is renewed. */ +export const snapshotArchiveCleanupLimits = Object.freeze({ lifetimeMs: 30000, attempts: 32, delayMs: 1000 }) + +function wait(milliseconds: number, signal: AbortSignal): Promise { + return new Promise(resolve => { + const finish = (): void => { + clearTimeout(timer) + signal.removeEventListener('abort', finish) + resolve() + } + const timer = setTimeout(finish, milliseconds) + signal.addEventListener('abort', finish, { once: true }) + // The private signal was checked immediately before this synchronous setup. + }) +} + +/** Bound polling and abort outstanding I/O, but await its settlement before releasing the caller. */ +export async function drainSnapshotArchiveRequest(cancel: (signal: AbortSignal) => Promise): Promise { + const startedAt = performance.now() + const expiresAt = Date.now() + snapshotArchiveCleanupLimits.lifetimeMs + const controller = new AbortController() + const timer = setTimeout(() => controller.abort(), snapshotArchiveCleanupLimits.lifetimeMs) + const remaining = (): number => + Math.min(expiresAt - Date.now(), snapshotArchiveCleanupLimits.lifetimeMs - (performance.now() - startedAt)) + const assertTime = (): void => { + if (controller.signal.aborted || remaining() <= 0) throw new SnapshotArchiveCleanupPendingError() + } + let complete = false + let delay = 100 + function* attempts() { + for (let attempt = 0; attempt < snapshotArchiveCleanupLimits.attempts && !complete; attempt++) yield attempt + } + try { + await runInSeries(attempts(), async () => { + assertTime() + try { + await retrySnapshotArchiveOperation(async () => { + assertTime() + await cancel(controller.signal) + }) + complete = true + } catch (error) { + if (!(error instanceof SnapshotArchiveCleanupPendingError)) throw error + assertTime() + await wait(Math.min(delay, remaining()), controller.signal) + delay = Math.min(delay * 2, snapshotArchiveCleanupLimits.delayMs) + } + }) + if (!complete) throw new SnapshotArchiveCleanupPendingError() + } finally { + clearTimeout(timer) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveClientBoundary.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveClientBoundary.test.ts new file mode 100644 index 000000000..d5fe4458d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveClientBoundary.test.ts @@ -0,0 +1,257 @@ +import { AuthFetch, PrivateKey, ProtoWallet } from '@bsv/sdk' +import { StorageClient } from '../../remoting/StorageClient' +import { BINARY_ENCODING, BINARY_ENCODING_HEADER } from '../../remoting/BinaryJson' +import { + snapshotArchiveCapabilities, + snapshotArchiveRequestBytes, + snapshotArchiveResponseBytes +} from './SnapshotArchiveProtocol' +import { snapshotHttpFixture } from '../../../../test/utils/snapshotArchiveHttpFixtures' + +afterEach(() => jest.restoreAllMocks()) + +test('server requires bounded compact framing and refuses its own oversized snapshot result', async () => { + const fixture = await snapshotHttpFixture() + try { + const { server, url } = await fixture.serve() + const auth = new AuthFetch(fixture.wallet) + for (const [binary, id, padding] of [ + [false, 1, ''], + [true, 0, ''], + [true, '1', ''], + [true, 1, 'x'.repeat(snapshotArchiveRequestBytes)] + ] as const) { + const response = await auth.fetch(url, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + ...(binary ? { [BINARY_ENCODING_HEADER]: BINARY_ENCODING } : {}) + }, + body: JSON.stringify({ + jsonrpc: '2.0', + method: 'getSnapshotArchiveOffer', + params: [{ version: 1, identityKey: fixture.identityKey }], + id, + padding + }) + }) + const json = await response.json() + expect(json.error.message).toContain('bounded compact-binary RPC framing') + } + const client = new StorageClient(fixture.wallet, url) + const transport = (await client.getSnapshotArchiveTransport(fixture.identityKey))! + const rpc = Reflect.get(server, 'snapshotArchives') + jest.spyOn(rpc, 'dispatch').mockResolvedValue({ padding: 'x'.repeat(snapshotArchiveResponseBytes) }) + await expect(transport.offer()).rejects.toThrow('response exceeds its transport limit') + } finally { + await fixture.close() + } +}) + +test('dedicated snapshot response cap is enforced before JSON parsing over authenticated HTTP', async () => { + const fixture = await snapshotHttpFixture() + try { + const { server, url } = await fixture.serve() + const client = new StorageClient(fixture.wallet, url) + const transport = (await client.getSnapshotArchiveTransport(fixture.identityKey))! + const send = Reflect.get(server, 'sendRpc').bind(server) + // An authenticated peer can be a different implementation: bypass only + // this fixture's snapshot-specific response bound, retaining real signing. + jest.spyOn(server as never, 'sendRpc' as never).mockImplementation((( + res: unknown, + binary: boolean, + payload: { id: unknown; result: unknown } + ) => { + const large = { + jsonrpc: '2.0', + id: payload.id, + result: { padding: 'x'.repeat(snapshotArchiveResponseBytes + 1) } + } + return send(res, binary, large) + }) as never) + await expect(transport.offer()).rejects.toThrow(/limit|large|size|exceed/i) + const bounded = Reflect.get(client, 'snapshotAuthClient') + expect(Reflect.get(bounded, 'maxResponseBytes')).toBe(snapshotArchiveResponseBytes) + } finally { + await fixture.close() + } +}) + +test.each(['no-auth', 'changed-server', 'no-binary', 'wrong-id', 'non-ok'] as const)( + 'snapshot calls retain the shared authenticated response contract: %s', + async variant => { + const fixture = await snapshotHttpFixture() + try { + const { url } = await fixture.serve() + const client = new StorageClient(fixture.wallet, url, { serverIdentityKey: fixture.serverIdentityKey }) + const transport = (await client.getSnapshotArchiveTransport(fixture.identityKey))! + const expectedId = Reflect.get(client, 'nextId') + const identity = + variant === 'changed-server' ? PrivateKey.fromRandom().toPublicKey().toString() : fixture.serverIdentityKey + const headers = { + ...(variant === 'no-auth' ? {} : { 'x-bsv-auth-identity-key': identity }), + ...(variant === 'no-binary' ? {} : { [BINARY_ENCODING_HEADER]: BINARY_ENCODING }) + } + jest + .spyOn(AuthFetch.prototype, 'fetch') + .mockResolvedValue( + new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: variant === 'wrong-id' ? expectedId + 1 : expectedId, + result: { + version: 1, + serverTime: Date.now(), + sourceStorageIdentityKey: 'http-snapshot-source', + sourceSchema: 'schema', + chain: 'test' + } + }), + { headers, status: variant === 'non-ok' ? 503 : 200 } + ) + ) + const reason = { + 'no-auth': /not mutually authenticated/, + 'changed-server': /identity changed/, + 'no-binary': /compact binary responses/, + 'wrong-id': /id/, + 'non-ok': /503/ + }[variant] + await expect(transport.offer()).rejects.toThrow(reason) + } finally { + await fixture.close() + } + } +) + +test('oversized snapshot envelopes reject before transport and advertised settings are privately detached', async () => { + const fixture = await snapshotHttpFixture() + try { + const { url } = await fixture.serve() + const client = new StorageClient(fixture.wallet, url) + const settings = await client.makeAvailable() + settings.storageIdentityKey = 'caller-mutated' + settings.chain = 'main' + settings.snapshotArchive = undefined + const transport = (await client.getSnapshotArchiveTransport(fixture.identityKey))! + expect((await transport.offer()).sourceStorageIdentityKey).toBe('http-snapshot-source') + const fetch = jest.spyOn(AuthFetch.prototype, 'fetch') + const call = Reflect.get(client, 'snapshotRpcCall').bind(client) + await expect( + call('getSnapshotArchiveOffer', [{ padding: 'x'.repeat(snapshotArchiveRequestBytes) }]) + ).rejects.toThrow('transport limit') + expect(fetch).not.toHaveBeenCalled() + } finally { + await fixture.close() + } +}) + +test('legacy settings decline the capability and malformed advertisements fail rather than fall back', async () => { + const wallet = new ProtoWallet(PrivateKey.fromRandom()) + for (const value of [undefined, null, { ...snapshotArchiveCapabilities, version: 2 }, snapshotArchiveCapabilities]) { + const client = new StorageClient(wallet, 'http://localhost:1') + jest.spyOn(client as never, 'rpcCall' as never).mockResolvedValue({ + storageIdentityKey: 'legacy', + ...(value === undefined ? {} : { snapshotArchive: value }) + } as never) + if (value !== undefined && value !== snapshotArchiveCapabilities) { + await expect(client.makeAvailable()).rejects.toThrow() + } else { + expect(await client.getSnapshotArchiveTransport(PrivateKey.fromRandom().toPublicKey().toString())).toBeUndefined() + } + } +}) + +test.each([ + [4095, snapshotArchiveResponseBytes, false], + [4096, snapshotArchiveResponseBytes, true], + [4096, -1, true], + [4096, snapshotArchiveResponseBytes - 1, false] +] as const)('capability respects body %p and response %p limits', async (bodyLimit, responseLimit, enabled) => { + const fixture = await snapshotHttpFixture() + const previous = process.env.WALLET_STORAGE_JSON_MAX_BODY_BYTES + try { + process.env.WALLET_STORAGE_JSON_MAX_BODY_BYTES = String(bodyLimit) + const { url } = await fixture.serve({ maxRpcResponseBytes: responseLimit }) + const client = new StorageClient(fixture.wallet, url) + const settings = await client.makeAvailable() + expect(settings.snapshotArchive).toEqual(enabled ? snapshotArchiveCapabilities : undefined) + } finally { + if (previous === undefined) delete process.env.WALLET_STORAGE_JSON_MAX_BODY_BYTES + else process.env.WALLET_STORAGE_JSON_MAX_BODY_BYTES = previous + await fixture.close() + } +}) + +test('main-chain client advertises a usable transport and admits an exactly bounded request', async () => { + const key = PrivateKey.fromRandom().toPublicKey().toString() + const client = new StorageClient(new ProtoWallet(PrivateKey.fromRandom()), 'http://localhost:1') + jest.spyOn(client as never, 'rpcCall' as never).mockResolvedValue({ + storageIdentityKey: 'main-source', + chain: 'main', + snapshotArchive: snapshotArchiveCapabilities + } as never) + const transport = (await client.getSnapshotArchiveTransport(key))! + const fetch = jest.spyOn(AuthFetch.prototype, 'fetch').mockImplementation(async (_url, options) => { + const input = JSON.parse(options!.body as string) + return new Response( + JSON.stringify({ + jsonrpc: '2.0', + id: input.id, + result: { + version: 1, + serverTime: Date.now(), + sourceStorageIdentityKey: 'main-source', + sourceSchema: 'schema', + chain: 'main' + } + }), + { headers: { 'x-bsv-auth-identity-key': key, [BINARY_ENCODING_HEADER]: BINARY_ENCODING } } + ) + }) + expect((await transport.offer()).chain).toBe('main') + const call = Reflect.get(client, 'snapshotRpcCall').bind(client) + const params = [{ padding: '' }] + const base = JSON.stringify({ + jsonrpc: '2.0', + method: 'getSnapshotArchiveOffer', + params, + id: Reflect.get(client, 'nextId') + }) + params[0].padding = 'x'.repeat(snapshotArchiveRequestBytes - Buffer.byteLength(base)) + await expect(call('getSnapshotArchiveOffer', params)).resolves.toMatchObject({ chain: 'main' }) + expect(Buffer.byteLength(fetch.mock.calls.at(-1)![1]!.body as string)).toBe(snapshotArchiveRequestBytes) +}) + +test('the server admits exact request/response ceilings, including request ID one', async () => { + const fixture = await snapshotHttpFixture() + try { + const { server, url } = await fixture.serve() + const auth = new AuthFetch(fixture.wallet) + const input = { + jsonrpc: '2.0', + method: 'getSnapshotArchiveOffer', + params: [{ version: 1, identityKey: fixture.identityKey }], + id: 1, + padding: '' + } + input.padding = 'x'.repeat(snapshotArchiveRequestBytes - Buffer.byteLength(JSON.stringify(input))) + const options = { + method: 'POST', + headers: { 'Content-Type': 'application/json', [BINARY_ENCODING_HEADER]: BINARY_ENCODING }, + body: JSON.stringify(input) + } + const accepted = await auth.fetch(url, options) + expect(await accepted.json()).toMatchObject({ jsonrpc: '2.0', id: 1, result: { version: 1, chain: 'test' } }) + const rpc = Reflect.get(server, 'snapshotArchives') + const base = JSON.stringify({ jsonrpc: '2.0', result: { padding: '' }, id: 1 }) + jest + .spyOn(rpc, 'dispatch') + .mockResolvedValue({ padding: 'x'.repeat(snapshotArchiveResponseBytes - Buffer.byteLength(base)) }) + const response = await auth.fetch(url, options) + expect(response.status).toBe(200) + expect(Buffer.byteLength(await response.text())).toBe(snapshotArchiveResponseBytes) + } finally { + await fixture.close() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts new file mode 100644 index 000000000..0f3185138 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.property.test.ts @@ -0,0 +1,68 @@ +import fc from 'fast-check' +import { verifySnapshotArchiveDirectory, verifySnapshotArchivePage } from './SnapshotArchiveDirectory' +import { expected, fixture, hash, now, rehash, tables } from '../../../../test/utils/snapshotArchiveDirectoryFixtures' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +function generatedDirectory(pageCounts: number[], seed: Uint8Array) { + const { directory } = fixture() + const payloads: Uint8Array[] = [] + const ranges: Array<{ first: number; pages: number; rows: number }> = [] + directory.receipts = [] + directory.rows = 0 + for (const [tableIndex, table] of tables.entries()) { + const range = { first: payloads.length, pages: pageCounts[tableIndex], rows: 0 } + for (let page = 0; page < range.pages; page++) { + const bytes = new Uint8Array([...seed, tableIndex, page]) + const done = page === range.pages - 1 + const rows = done ? seed[0] % 10 : 1 + seed[0] + directory.receipts.push({ sequence: payloads.length, table, rows, done, digest: hash(bytes) }) + payloads.push(bytes) + range.rows += rows + } + directory.rows += range.rows + ranges.push(range) + } + directory.pages = payloads.length + rehash(directory) + return { directory, payloads, ranges } +} + +test('generated receipt chains support arbitrary table reads while rejecting changed page and manifest bytes', () => { + fc.assert( + fc.property( + fc.array(fc.integer({ min: 1, max: 4 }), { minLength: 13, maxLength: 13 }), + fc.uint8Array({ minLength: 1, maxLength: 16 }), + fc.integer({ min: 0, max: 1000 }), + (pageCounts, seed, selection) => { + const { directory, payloads, ranges } = generatedDirectory(pageCounts, seed) + const verified = verifySnapshotArchiveDirectory(directory, { ...expected, digest: directory.digest }, now) + expect(verified.manifest.rows).toBe(ranges.reduce((total, range) => total + range.rows, 0)) + for (const [index, table] of tables.entries()) expect(verified.tables[table]).toEqual(ranges[index]) + const selected = selection % payloads.length + const original = payloads[selected] + const page = { ...directory.receipts[selected], bytes: original } + expect(verifySnapshotArchivePage(page, verified.receipts[selected])).toEqual(original) + const changed = new Uint8Array(original) + changed[0] ^= 1 + expect(() => verifySnapshotArchivePage({ ...page, bytes: changed }, verified.receipts[selected])).toThrow() + directory.receipts[selected].digest = hash(changed) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow() + rehash(directory) + expect(() => + verifySnapshotArchiveDirectory(directory, { ...expected, digest: verified.manifest.digest }, now) + ).toThrow() + directory.receipts[selected].rows++ + expect(verified.receipts[selected].rows).toBe(page.rows) + } + ) + ) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.test.ts new file mode 100644 index 000000000..26c25b69f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.test.ts @@ -0,0 +1,376 @@ +import { + snapshotArchiveDirectoryBytes, + verifySnapshotArchiveDirectory, + verifySnapshotArchivePage +} from './SnapshotArchiveDirectory' +import { + expected, + fixture, + hash, + now, + rehash, + sourceStorageIdentityKey +} from '../../../../test/utils/snapshotArchiveDirectoryFixtures' + +test.each([null, undefined, 1, 'directory', []])('rejects a non-record directory %p', value => { + expect(() => verifySnapshotArchiveDirectory(value, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('uses the current clock by default and accepts the exact maximum lifetime', () => { + const { directory } = fixture() + directory.expiresAt = now + 3600000 + const clock = jest.spyOn(Date, 'now').mockReturnValue(now) + try { + expect(verifySnapshotArchiveDirectory(directory, expected).manifest.expiresAt).toBe(now + 3600000) + } finally { + clock.mockRestore() + } +}) + +test.each(['not-a-date', '2026-09-30', '2026-09-30T00:00:00+00:00'])('rejects noncanonical source date %s', value => { + const { directory, binding } = fixture() + binding.sourceStorage.created_at = value + directory.bindingJson = JSON.stringify(binding) + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test.each(['version', 'database', 'script-limit', 'user-id', 'schema-size'])('rejects invalid binding %s', variant => { + const { directory, binding } = fixture() + if (variant === 'version') binding.version = 2 + if (variant === 'database') binding.sourceStorage.dbtype = 'IndexedDB' + if (variant === 'script-limit') binding.sourceStorage.maxOutputScript = -1 + if (variant === 'user-id') binding.user.userId = 0 + if (variant === 'schema-size') binding.sourceSchema = 'x'.repeat(257) + directory.bindingJson = JSON.stringify(binding) + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('accepts original MySQL metadata with empty permitted labels', () => { + const { directory, binding } = fixture() + binding.sourceStorage.dbtype = 'MySQL' + binding.sourceStorage.storageName = '' + binding.user.activeStorage = '' + binding.sourceStorage.maxOutputScript = 0 + directory.bindingJson = JSON.stringify(binding) + rehash(directory) + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + expect(verified.manifest.binding.sourceStorage).toMatchObject({ + dbtype: 'MySQL', + storageName: '', + maxOutputScript: 0 + }) + expect(verified.manifest.binding.user.activeStorage).toBe('') +}) + +test('rejects oversized UTF-8 metadata before its JSON is interpreted', () => { + const { directory, binding } = fixture() + binding.sourceStorage.storageName = 'é'.repeat(40000) + directory.bindingJson = JSON.stringify(binding) + expect(directory.bindingJson.length).toBeLessThan(65536) + expect(new TextEncoder().encode(directory.bindingJson).length).toBeGreaterThan(65536) + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('rejects malformed UTF-16 metadata rather than hashing replacement text', () => { + const { directory } = fixture() + directory.bindingJson = directory.bindingJson.replace('Original source', '\ud800') + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('all declared fields must be own enumerable data', () => { + const { directory } = fixture() + const hidden = { ...directory } + Object.defineProperty(hidden, 'digest', { value: directory.digest, enumerable: false }) + expect(() => verifySnapshotArchiveDirectory(hidden, expected, now)).toThrow('Invalid snapshot archive directory') + const renamed = { ...directory, replacement: directory.digest } as Record + delete renamed.digest + expect(() => verifySnapshotArchiveDirectory(renamed, expected, now)).toThrow('Invalid snapshot archive directory') + const symbol = { ...directory, [Symbol('extra')]: true } + expect(() => verifySnapshotArchiveDirectory(symbol, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('a correctly hashed but incomplete final table remains invalid', () => { + const { directory } = fixture() + directory.receipts[12].done = false + directory.receipts[12].rows = 1 + directory.rows = 1 + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('a correctly hashed additional table remains invalid', () => { + const { directory } = fixture() + directory.receipts.push({ ...directory.receipts[12], sequence: 13 }) + directory.pages++ + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('preserves original binding bytes and verifies every table before out-of-order reads', () => { + const { directory, payloads } = fixture(2) + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + expect(verified.bindingJson).toBe(directory.bindingJson) + expect(verified.manifest.binding.sourceStorage.storageIdentityKey).toBe(sourceStorageIdentityKey) + expect(verified.manifest.binding.user.activeStorage).toBe('historical-primary') + expect(verified.tables.provenTxs).toEqual({ first: 0, pages: 3, rows: 2 }) + expect(verified.tables.syncStates).toEqual({ first: 14, pages: 1, rows: 0 }) + for (const index of [14, 0, 7]) { + const page = { ...directory.receipts[index], bytes: payloads[index] } + const copied = verifySnapshotArchivePage(page, verified.receipts[index]) + expect(copied).toEqual(payloads[index]) + expect(copied).not.toBe(payloads[index]) + } +}) + +test('the binding hash uses exact UTF-8 bytes rather than parse/reserialize order', () => { + const { directory, binding } = fixture() + directory.bindingJson = JSON.stringify( + { + user: binding.user, + sourceSchema: binding.sourceSchema, + sourceStorage: binding.sourceStorage, + snapshotId: binding.snapshotId, + version: binding.version + }, + null, + 1 + ) + rehash(directory) + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + expect(verified.bindingJson).toBe(directory.bindingJson) + const compact = { ...directory, bindingJson: JSON.stringify(JSON.parse(directory.bindingJson)) } + expect(() => verifySnapshotArchiveDirectory(compact, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test('caller mutations cannot replace verified receipt or table positions', () => { + const { directory } = fixture() + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + directory.receipts[0].digest = 'f'.repeat(64) + directory.receipts.reverse() + expect(verified.receipts[0].table).toBe('provenTxs') + expect(verified.receipts[0].digest).not.toBe('f'.repeat(64)) + expect(Object.isFrozen(verified)).toBe(true) + expect(Object.isFrozen(verified.receipts)).toBe(true) + expect(Object.isFrozen(verified.receipts[0])).toBe(true) + expect(Object.isFrozen(verified.tables)).toBe(true) + expect(Object.isFrozen(verified.tables.provenTxs)).toBe(true) +}) + +test.each([ + ['version', 2], + ['encoding', 'other'], + ['archiveId', 'B'.repeat(64)], + ['expiresAt', now], + ['expiresAt', now + 3600001], + ['pages', 12], + ['pages', 4097], + ['pages', 13.5], + ['rows', -1], + ['rows', Number.MAX_SAFE_INTEGER], + ['digest', 'z'.repeat(64)], + ['bindingJson', ''], + ['receipts', null] +])('rejects invalid directory %s=%s', (key, value) => { + const { directory } = fixture() + expect(() => verifySnapshotArchiveDirectory({ ...directory, [key]: value }, expected, now)).toThrow( + 'Invalid snapshot archive directory' + ) +}) + +test.each([NaN, Infinity, -1, 0.5])('rejects invalid verifier clock %s', clock => { + expect(() => verifySnapshotArchiveDirectory(fixture().directory, expected, clock)).toThrow( + 'Invalid snapshot archive directory' + ) +}) + +test.each([ + { identityKey: '03' + '22'.repeat(32) }, + { chain: 'main' as const }, + { sourceStorageIdentityKey: 'replacement' }, + { sourceSchema: 'replacement' }, + { archiveId: 'c'.repeat(64) }, + { digest: 'd'.repeat(64) } +])('rejects changed expected profile/source or resumed binding %p', changed => { + expect(() => verifySnapshotArchiveDirectory(fixture().directory, { ...expected, ...changed }, now)).toThrow( + 'Invalid snapshot archive directory' + ) +}) + +test('a resumed handle can require its already-observed root and schema', () => { + const { directory, binding } = fixture() + expect( + verifySnapshotArchiveDirectory( + directory, + { ...expected, archiveId: directory.archiveId, digest: directory.digest, sourceSchema: binding.sourceSchema }, + now + ).manifest.digest + ).toBe(directory.digest) +}) + +test.each(['version', 'snapshotId', 'sourceSchema', 'sourceStorage', 'user'])( + 'refuses missing binding field %s', + key => { + const { directory, binding } = fixture() + const copy = { ...binding } as Record + delete copy[key] + directory.bindingJson = JSON.stringify(copy) + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') + } +) + +test('rejects extra internal credentials without reading accessor values', () => { + const { directory } = fixture() + expect(() => verifySnapshotArchiveDirectory({ ...directory, writerToken: 'internal' }, expected, now)).toThrow( + 'Invalid snapshot archive directory' + ) + let reads = 0 + const withAccessor = { ...directory } + Object.defineProperty(withAccessor, 'digest', { + enumerable: true, + get: () => { + reads++ + return directory.digest + } + }) + expect(() => verifySnapshotArchiveDirectory(withAccessor, expected, now)).toThrow( + 'Invalid snapshot archive directory' + ) + expect(reads).toBe(0) + const receipts = [...directory.receipts] + Object.defineProperty(receipts, 0, { + get: () => { + reads++ + return directory.receipts[0] + } + }) + expect(() => verifySnapshotArchiveDirectory({ ...directory, receipts }, expected, now)).toThrow( + 'Invalid snapshot archive directory' + ) + expect(reads).toBe(0) +}) + +test.each(['omitted', 'reordered', 'unfinished', 'wrong-table', 'wrong-sequence', 'wrong-total', 'different-root'])( + 'rejects %s receipt chains', + variant => { + const { directory } = fixture() + if (variant === 'omitted') delete directory.receipts[0] + if (variant === 'reordered') directory.receipts.reverse() + if (variant === 'unfinished') directory.receipts[0].done = false + if (variant === 'wrong-table') directory.receipts[0].table = 'outputs' + if (variant === 'wrong-sequence') directory.receipts[0].sequence = 1 + if (variant === 'wrong-total') directory.rows = 1 + if (variant === 'different-root') directory.receipts[0].digest = 'c'.repeat(64) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') + } +) + +test('the maximum receipt count remains bounded below the transport metadata budget', () => { + const { directory, binding } = fixture(4096 - 13) + const base = new TextEncoder().encode(JSON.stringify(binding)).length + binding.sourceStorage.storageName = '\u0001'.repeat( + Math.floor((65536 - base + binding.sourceStorage.storageName.length) / 6) + ) + binding.sourceStorage.storageName += 'x'.repeat(65536 - new TextEncoder().encode(JSON.stringify(binding)).length) + directory.bindingJson = JSON.stringify(binding) + expect(new TextEncoder().encode(directory.bindingJson)).toHaveLength(65536) + rehash(directory) + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + expect(verified.receipts).toHaveLength(4096) + expect(new TextEncoder().encode(JSON.stringify(directory)).length).toBeLessThan(snapshotArchiveDirectoryBytes) +}) + +test('accepts the exact byte and row limits without changing copied page bytes', () => { + const { directory } = fixture() + const bytes = new Uint8Array(1024 * 1024).fill(7) + directory.receipts[0].digest = hash(bytes) + directory.receipts[0].rows = 1000 + directory.rows = 1000 + rehash(directory) + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + const received = verifySnapshotArchivePage({ ...directory.receipts[0], bytes }, verified.receipts[0]) + expect(received).toHaveLength(1024 * 1024) + expect(hash(received)).toBe(hash(bytes)) + expect(verified.tables.provenTxs.rows).toBe(1000) +}) + +test.each(['metadata', 'bytes', 'digest', 'extra', 'empty', 'oversized'])( + 'rejects %s page substitution against the verified receipt', + variant => { + const { directory, payloads } = fixture() + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + const page: Record = { ...directory.receipts[0], bytes: payloads[0] } + if (variant === 'metadata') page.table = 'outputs' + if (variant === 'bytes') page.bytes = payloads[1] + if (variant === 'digest') page.digest = 'c'.repeat(64) + if (variant === 'extra') page.writerToken = 'internal' + if (variant === 'empty') page.bytes = new Uint8Array() + if (variant === 'oversized') page.bytes = new Uint8Array(1024 * 1024 + 1) + expect(() => verifySnapshotArchivePage(page, verified.receipts[0])).toThrow('Invalid snapshot archive directory') + } +) + +test.each([0, null, { length: 12 }, ''])('refuses a non-text or empty source schema %p', value => { + const { directory, binding } = fixture() + directory.bindingJson = JSON.stringify({ ...binding, sourceSchema: value }) + rehash(directory) + expect(() => verifySnapshotArchiveDirectory(directory, expected, now)).toThrow('Invalid snapshot archive directory') +}) + +test.each(['count', 'array-like', 'done-type', 'empty-continuation'])( + 'refuses correctly hashed invalid receipt structure %s', + variant => { + const { directory } = fixture(1) + if (variant === 'count') directory.receipts.push({ ...directory.receipts[13], sequence: 14 }) + if (variant === 'done-type') Object.assign(directory.receipts[0], { done: 0 }) + if (variant === 'empty-continuation') { + directory.receipts[0].rows = 0 + directory.rows = 0 + } + rehash(directory) + const input = + variant === 'array-like' + ? { ...directory, receipts: { ...directory.receipts, length: directory.pages } } + : directory + expect(() => verifySnapshotArchiveDirectory(input, expected, now)).toThrow('Invalid snapshot archive directory') + } +) + +test.each(['sequence', 'rows', 'done'])('refuses changed page %s even when its payload digest is correct', field => { + const { directory, payloads } = fixture() + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + const page = { ...directory.receipts[0], bytes: payloads[0] } + const changed = field === 'done' ? false : 1 + expect(() => verifySnapshotArchivePage({ ...page, [field]: changed }, verified.receipts[0])).toThrow( + 'Invalid snapshot archive directory' + ) +}) + +test.each(['array', 'array-like', 'empty', 'oversized'])( + 'requires bounded typed page bytes with a matching digest: %s', + variant => { + const bytes = + variant === 'empty' ? new Uint8Array() : new Uint8Array(variant === 'oversized' ? 1024 * 1024 + 1 : 1).fill(9) + const { directory } = fixture() + directory.receipts[0].digest = hash(bytes) + rehash(directory) + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + const input = variant === 'array' ? Array.from(bytes) : variant === 'array-like' ? { 0: 9, length: 1 } : bytes + expect(() => verifySnapshotArchivePage({ ...directory.receipts[0], bytes: input }, verified.receipts[0])).toThrow( + 'Invalid snapshot archive directory' + ) + } +) + +test('accepts one byte with the digest bound into the directory', () => { + const bytes = Uint8Array.of(9) + const { directory } = fixture() + directory.receipts[0].digest = hash(bytes) + rehash(directory) + const verified = verifySnapshotArchiveDirectory(directory, expected, now) + expect(verifySnapshotArchivePage({ ...directory.receipts[0], bytes }, verified.receipts[0])).toEqual(bytes) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.ts new file mode 100644 index 000000000..17e838007 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveDirectory.ts @@ -0,0 +1,275 @@ +import { syncTransferDigest } from '../../remoting/SyncTransfer' +import type { WalletSnapshotTable } from '../WalletReadSnapshot' +import { + snapshotArchiveLimits, + snapshotArchiveTables, + type SnapshotArchiveBinding, + type SnapshotArchiveManifest, + type SnapshotArchivePage +} from './SnapshotArchive' + +export const snapshotArchiveEncoding = 'wallet-snapshot-rows/1' +export const snapshotArchiveDirectoryBytes = 1024 * 1024 + +export type SnapshotArchiveReceipt = Omit + +/** The exact binding JSON is the hash preimage, not a reserialized object. */ +export interface SnapshotArchiveDirectory { + version: 1 + encoding: typeof snapshotArchiveEncoding + archiveId: string + expiresAt: number + pages: number + rows: number + digest: string + bindingJson: string + receipts: SnapshotArchiveReceipt[] +} + +export interface SnapshotArchiveExpectedSource { + identityKey: string + chain: 'main' | 'test' + sourceStorageIdentityKey: string + archiveId?: string + digest?: string + sourceSchema?: string +} + +export interface SnapshotArchiveTableRange { + first: number + pages: number + rows: number +} + +export interface VerifiedSnapshotArchiveDirectory { + manifest: SnapshotArchiveManifest + bindingJson: string + receipts: ReadonlyArray> + tables: Readonly>> +} + +function invalid(): never { + throw new TypeError('Invalid snapshot archive directory') +} + +function record(value: unknown, fields: string[]): Record { + if (value === null || typeof value !== 'object' || Array.isArray(value)) invalid() + const keys = Reflect.ownKeys(value) + if (keys.length !== fields.length) invalid() + const result: Record = {} + for (const field of fields) { + const property = Object.getOwnPropertyDescriptor(value, field) + if (property === undefined || !('value' in property) || property.enumerable !== true) invalid() + result[field] = property.value + } + return result +} + +function integer(value: unknown, minimum: number, maximum: number): number { + if (!Number.isSafeInteger(value) || (value as number) < minimum || (value as number) > maximum) invalid() + return value as number +} + +function text(value: unknown, maximum: number, minimum = 1): string { + if (typeof value !== 'string' || value.length < minimum || value.length > maximum) invalid() + return value +} + +function digest(value: unknown): string { + const result = text(value, 64) + if (!/^[0-9a-f]{64}$/.test(result)) invalid() + return result +} + +function date(value: unknown): Date { + const result = new Date(text(value, 32)) + if (!Number.isFinite(result.getTime()) || result.toISOString() !== value) invalid() + return result +} + +function binding(encoded: string, expected: SnapshotArchiveExpectedSource): SnapshotArchiveBinding { + const bytes = new TextEncoder().encode(encoded) + if (bytes.length > snapshotArchiveLimits.bindingBytes || new TextDecoder().decode(bytes) !== encoded) invalid() + const value = record(JSON.parse(encoded), ['version', 'snapshotId', 'sourceStorage', 'sourceSchema', 'user']) + if (value.version !== 1) invalid() + const storage = record(value.sourceStorage, [ + 'created_at', + 'updated_at', + 'storageIdentityKey', + 'storageName', + 'chain', + 'dbtype', + 'maxOutputScript' + ]) + const user = record(value.user, ['created_at', 'updated_at', 'userId', 'identityKey', 'activeStorage']) + if (expected.sourceSchema !== undefined && value.sourceSchema !== expected.sourceSchema) invalid() + if ( + storage.chain !== expected.chain || + storage.storageIdentityKey !== expected.sourceStorageIdentityKey || + user.identityKey !== expected.identityKey + ) + invalid() + if (storage.dbtype !== 'SQLite' && storage.dbtype !== 'MySQL') invalid() + return { + version: 1, + snapshotId: digest(value.snapshotId), + sourceSchema: text(value.sourceSchema, 256), + sourceStorage: { + created_at: date(storage.created_at), + updated_at: date(storage.updated_at), + storageIdentityKey: text(storage.storageIdentityKey, 130), + storageName: text(storage.storageName, snapshotArchiveLimits.bindingBytes, 0), + chain: expected.chain, + dbtype: storage.dbtype, + maxOutputScript: integer(storage.maxOutputScript, 0, Number.MAX_SAFE_INTEGER) + }, + user: { + created_at: date(user.created_at), + updated_at: date(user.updated_at), + userId: integer(user.userId, 1, Number.MAX_SAFE_INTEGER), + identityKey: text(user.identityKey, 66), + activeStorage: text(user.activeStorage, 130, 0) + } + } +} + +interface ValidatedDirectoryHeader { + archiveId: string + expiresAt: number + pages: number + rows: number + digest: string + bindingJson: string + source: SnapshotArchiveBinding + receipts: unknown[] +} + +function directoryHeader( + input: unknown, + expected: SnapshotArchiveExpectedSource, + now: number +): ValidatedDirectoryHeader { + const value = record(input, [ + 'version', + 'encoding', + 'archiveId', + 'expiresAt', + 'pages', + 'rows', + 'digest', + 'bindingJson', + 'receipts' + ]) + if (value.version !== 1 || value.encoding !== snapshotArchiveEncoding) invalid() + const archiveId = digest(value.archiveId) + if (expected.archiveId !== undefined && archiveId !== expected.archiveId) invalid() + const expiresAt = integer(value.expiresAt, 1, Number.MAX_SAFE_INTEGER) + integer(now, 0, Number.MAX_SAFE_INTEGER) + if (expiresAt <= now || expiresAt - now > snapshotArchiveLimits.lifetimeMs) invalid() + const pages = integer(value.pages, snapshotArchiveTables.length, snapshotArchiveLimits.pages) + const rows = integer(value.rows, 0, snapshotArchiveLimits.pages * snapshotArchiveLimits.rowsPerPage) + const root = digest(value.digest) + if (expected.digest !== undefined && root !== expected.digest) invalid() + const bindingJson = text(value.bindingJson, snapshotArchiveLimits.bindingBytes) + const source = binding(bindingJson, expected) + if (!Array.isArray(value.receipts) || value.receipts.length !== pages) invalid() + return { archiveId, expiresAt, pages, rows, digest: root, bindingJson, source, receipts: value.receipts } +} + +function pageReceipt( + input: unknown[], + index: number, + table: WalletSnapshotTable | undefined +): Readonly { + const slot = Object.getOwnPropertyDescriptor(input, index) + if (slot === undefined || !('value' in slot)) invalid() + const raw = record(slot.value, ['sequence', 'table', 'rows', 'done', 'digest']) + if (table === undefined || raw.sequence !== index || raw.table !== table || typeof raw.done !== 'boolean') invalid() + const rows = integer(raw.rows, 0, snapshotArchiveLimits.rowsPerPage) + if (rows === 0 && !raw.done) invalid() + return Object.freeze({ sequence: index, table, rows, done: raw.done, digest: digest(raw.digest) }) +} + +function receiptDirectory( + header: ValidatedDirectoryHeader +): Pick { + let previous = syncTransferDigest(new TextEncoder().encode(header.bindingJson)) + let tableIndex = 0 + let totalRows = 0 + const receipts: Array> = [] + const tables: Partial>> = {} + let first = 0 + let tableRows = 0 + for (let index = 0; index < header.pages; index++) { + const receipt = pageReceipt(header.receipts, index, snapshotArchiveTables[tableIndex]) + previous = syncTransferDigest( + new TextEncoder().encode( + JSON.stringify([previous, index, receipt.table, receipt.rows, receipt.done, receipt.digest]) + ) + ) + receipts.push(receipt) + totalRows += receipt.rows + tableRows += receipt.rows + if (receipt.done) { + tables[receipt.table] = Object.freeze({ first, pages: index - first + 1, rows: tableRows }) + first = index + 1 + tableRows = 0 + tableIndex++ + } + } + if (tableIndex !== snapshotArchiveTables.length || totalRows !== header.rows || previous !== header.digest) invalid() + return { + receipts: Object.freeze(receipts), + tables: Object.freeze(tables) as VerifiedSnapshotArchiveDirectory['tables'] + } +} + +/** Verify the complete bounded receipt chain before requesting arbitrary tables. */ +export function verifySnapshotArchiveDirectory( + input: unknown, + expected: SnapshotArchiveExpectedSource, + now = Date.now() +): VerifiedSnapshotArchiveDirectory { + const header = directoryHeader(input, expected, now) + const directory = receiptDirectory(header) + // Field and receipt limits bound the reconstructed metadata below one MiB. + // The transport separately caps bytes before parsing the response envelope. + Object.freeze(header.source.sourceStorage) + Object.freeze(header.source.user) + Object.freeze(header.source) + return Object.freeze({ + manifest: Object.freeze({ + version: 1 as const, + archiveId: header.archiveId, + expiresAt: header.expiresAt, + pages: header.pages, + rows: header.rows, + digest: header.digest, + binding: header.source + }), + bindingJson: header.bindingJson, + ...directory + }) +} + +/** Validate bytes against the already-verified directory, not a peer-supplied digest alone. */ +export function verifySnapshotArchivePage(input: unknown, receipt: Readonly): Uint8Array { + const value = record(input, ['sequence', 'table', 'rows', 'done', 'digest', 'bytes']) + if ( + value.sequence !== receipt.sequence || + value.table !== receipt.table || + value.rows !== receipt.rows || + value.done !== receipt.done || + value.digest !== receipt.digest + ) + invalid() + if ( + !(value.bytes instanceof Uint8Array) || + value.bytes.length < 1 || + value.bytes.length > snapshotArchiveLimits.pageBytes + ) + invalid() + const bytes = new Uint8Array(value.bytes) + if (syncTransferDigest(bytes) !== receipt.digest) invalid() + return bytes +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuard.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuard.test.ts new file mode 100644 index 000000000..9fd2307f4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuard.test.ts @@ -0,0 +1,573 @@ +import { mkdtemp, readdir, rename, rm, stat, symlink } from 'node:fs/promises' +import filesystem from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex, type Knex } from 'knex' +import { addSnapshotArchiveTables } from '../../schema/snapshotArchiveMigration' +import { addSnapshotArchiveRequestTable } from '../../schema/snapshotArchiveRequestMigration' +import { addSnapshotArchiveOwnerTable } from '../../schema/snapshotArchiveOwnerMigration' +import { + addSnapshotArchiveGuardTable, + removeSnapshotArchiveGuardTable +} from '../../schema/snapshotArchiveGuardMigration' +import { KnexSnapshotArchiveRequestStore } from './KnexSnapshotArchiveRequestStore' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import { readGuardedSnapshotArchive, recoverSnapshotArchiveGuards } from './SnapshotArchiveGuard' +import { SnapshotArchiveSourceCleanupError } from './KnexSnapshotArchiveSource' +import * as Backend from './SnapshotArchiveGuardBackend' +import * as Registry from './SnapshotArchiveGuardRegistry' +import * as ArchiveSql from './SnapshotArchiveSql' +import { reserveSnapshotArchiveOwner } from './SnapshotArchiveOwner' + +const stores: Knex[] = [] +const directories: string[] = [] +function gate() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} +function identity(index = 1): string { + return '02' + index.toString(16).padStart(2, '0').repeat(32) +} +function request(index = 1) { + const fields = { + version: 1 as const, + nonce: index.toString(16).padStart(2, '0').repeat(32), + notAfter: Date.now() + 300000, + maxBytes: 32768 + } + return { ...fields, requestId: snapshotArchiveRequestId(fields) } +} +async function fixture(upgrade = true) { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-guard-')) + directories.push(directory) + const filename = join(directory, 'wallet.sqlite') + const config: Knex.Config = { + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + } + const open = () => { + const value = knex(config) + stores.push(value) + return value + } + const control = open() + await control.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(control) + await addSnapshotArchiveRequestTable(control) + await addSnapshotArchiveOwnerTable(control) + if (upgrade) await addSnapshotArchiveGuardTable(control) + await control.schema.createTable('fixture_values', table => { + table.integer('id').primary() + table.string('value') + }) + await control('fixture_values').insert({ id: 1, value: 'before' }) + const requests = new KnexSnapshotArchiveRequestStore(control, true, upgrade) + const recover = async () => { + await recoverSnapshotArchiveGuards(control, config) + await requests.reap() + } + return { control, requests, recover, directory, filename, config, open } +} +afterEach(async () => { + jest.restoreAllMocks() + await Promise.all(stores.splice(0).map(value => value.destroy())) + await Promise.all(directories.splice(0).map(directory => rm(directory, { recursive: true, force: true }))) +}) + +test('the additive migration preserves unguarded owners and refuses their downgrade', async () => { + const f = await fixture(false) + const input = request() + const claimed = await f.requests.claim(identity(), input) + await addSnapshotArchiveGuardTable(f.control) + await addSnapshotArchiveGuardTable(f.control) + expect(await f.control('snapshot_archive_owner_slots')).toHaveLength(8) + expect(await f.control('snapshot_archive_owners').first()).toMatchObject({ ...claimed.owner, guardVersion: 0 }) + await f.requests.markCancellation(identity(), input) + await f.recover() + expect(await f.control('snapshot_archive_owners')).toHaveLength(1) + await expect(removeSnapshotArchiveGuardTable(f.control)).rejects.toThrow('Drain') + await f.requests.sourceClosed(claimed.owner!) + await removeSnapshotArchiveGuardTable(f.control) + await removeSnapshotArchiveGuardTable(f.control) + expect(await f.control.schema.hasColumn('snapshot_archive_owners', 'guardVersion')).toBe(false) +}) + +test('recovery waits through physical connection close while foreground WAL writes continue', async () => { + const f = await fixture() + const input = request() + const { owner } = await f.requests.claim(identity(), input) + const source = f.open(), + opened = gate(), + stop = gate(), + closing = gate(), + finish = gate() + const destroy = source.client.destroyRawConnection.bind(source.client) + jest.spyOn(source.client, 'destroyRawConnection').mockImplementation(async connection => { + closing.resolve() + await finish.promise + await destroy(connection) + }) + const work = readGuardedSnapshotArchive(f.control, source, owner!, async trx => { + expect((await trx('fixture_values').first()).value).toBe('before') + await expect(trx('fixture_values').update({ value: 'forbidden' })).rejects.toMatchObject({ + code: 'SQLITE_READONLY' + }) + opened.resolve() + await stop.promise + expect((await trx('fixture_values').first()).value).toBe('before') + }) + void work.catch(() => undefined) + try { + await opened.promise + await f.requests.markCancellation(identity(), input) + await f.recover() + expect(await f.control('snapshot_archive_owners')).toHaveLength(1) + await f.control('fixture_values').update({ value: 'foreground' }) + stop.resolve() + await closing.promise + await f.recover() + expect((await f.control('snapshot_archive_capacity').first()).archives).toBe(1) + finish.resolve() + await work + await f.recover() + expect(await f.control('snapshot_archive_owners')).toHaveLength(0) + expect((await f.control('snapshot_archive_capacity').first()).archives).toBe(0) + expect((await f.control('fixture_values').first()).value).toBe('foreground') + } finally { + stop.resolve() + finish.resolve() + await work.catch(() => undefined) + } +}) + +test.each(['unbound', 'bound'] as const)( + 'a %s late acquisition cannot read after recovery or release a successor slot', + async phase => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + const entered = gate(), + resume = gate() + if (phase === 'unbound') { + const bind = Registry.bindSnapshotArchiveOwnerGuard + jest.spyOn(Registry, 'bindSnapshotArchiveOwnerGuard').mockImplementationOnce(async (...args) => { + entered.resolve() + await resume.promise + return await bind(...args) + }) + } else { + const guard = Backend.withSnapshotArchiveBackendGuard + jest.spyOn(Backend, 'withSnapshotArchiveBackendGuard').mockImplementationOnce(async (...args) => { + entered.resolve() + await resume.promise + return await guard(...args) + }) + } + let reads = 0 + const work = readGuardedSnapshotArchive(f.control, f.open(), owner!, async () => { + reads++ + }) + void work.catch(() => undefined) + try { + await entered.promise + await f.requests.markCancellation(identity(), input) + await f.recover() + const successor = await f.requests.claim(identity(), request(2)) + resume.resolve() + await expect(work).rejects.toThrow('unavailable') + expect(reads).toBe(0) + await f.requests.sourceClosed(owner!) + expect(await f.control('snapshot_archive_owners').first()).toMatchObject(successor.owner!) + await readGuardedSnapshotArchive(f.control, f.open(), successor.owner!, async trx => { + expect((await trx('fixture_values').first()).value).toBe('before') + }) + await f.requests.sourceClosed(successor.owner!) + } finally { + resume.resolve() + await work.catch(() => undefined) + } + } +) + +test('a missing bound guard fails closed without recreating its inode', async () => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + await readGuardedSnapshotArchive(f.control, f.open(), owner!, async () => undefined) + await f.requests.markCancellation(identity(), input) + const path = `${f.filename}.snapshot-owner-0.sqlite`, + moved = path + '.retained' + const inode = (await stat(path)).ino + await rename(path, moved) + await expect(f.recover()).rejects.toMatchObject({ code: 'ENOENT' }) + expect(await readdir(f.directory)).not.toContain('wallet.sqlite.snapshot-owner-0.sqlite') + expect((await f.control('snapshot_archive_capacity').first()).archives).toBe(1) + await rename(moved, path) + await f.recover() + expect((await stat(path)).ino).toBe(inode) + expect((await f.control('snapshot_archive_capacity').first()).archives).toBe(0) +}) + +test('an unproved native close never rolls back the guard or releases source capacity', async () => { + const f = await fixture(), + input = request(), + source = f.open() + const { owner } = await f.requests.claim(identity(), input) + let connection: { open: boolean; close: () => void } | undefined + jest.spyOn(source.client, 'destroyRawConnection').mockImplementation(async value => { + connection = value + }) + try { + await expect( + readGuardedSnapshotArchive(f.control, source, owner!, async trx => { + expect((await trx('fixture_values').first()).value).toBe('before') + }) + ).rejects.toBeInstanceOf(SnapshotArchiveSourceCleanupError) + expect(connection?.open).toBe(true) + await f.requests.markCancellation(identity(), input) + await f.recover() + expect((await f.control('snapshot_archive_capacity').first()).archives).toBe(1) + connection!.close() + await f.recover() + expect((await f.control('snapshot_archive_capacity').first()).archives).toBe(0) + } finally { + if (connection?.open) connection.close() + } +}) + +test('all eight independent source guards preserve old views and stable slot files through reuse', async () => { + const f = await fixture(), + stop = gate(), + work: Promise[] = [] + try { + for (let index = 1; index <= 8; index++) { + const claimed = await f.requests.claim(identity(index), request(index)) + const opened = gate() + work.push( + readGuardedSnapshotArchive(f.control, f.open(), claimed.owner!, async trx => { + expect((await trx('fixture_values').first()).value).toBe('before') + opened.resolve() + await stop.promise + expect((await trx('fixture_values').first()).value).toBe('before') + }) + ) + await opened.promise + } + await expect(f.requests.claim(identity(9), request(9))).rejects.toThrow('capacity') + const files = (await readdir(f.directory)).filter(name => /snapshot-owner-\d.sqlite$/.test(name)).sort() + expect(files).toHaveLength(8) + const inodes = await Promise.all(files.map(async name => (await stat(join(f.directory, name))).ino)) + await f.control('fixture_values').update({ value: 'committed' }) + await f.control('snapshot_archive_requests').update({ expiresAt: 1 }) + await f.recover() + expect(await f.control('snapshot_archive_owners')).toHaveLength(8) + stop.resolve() + await Promise.all(work) + await f.recover() + expect((await f.control('snapshot_archive_capacity').first()).archives).toBe(0) + const successor = await f.requests.claim(identity(), request(10)) + await readGuardedSnapshotArchive(f.control, f.open(), successor.owner!, async trx => { + expect((await trx('fixture_values').first()).value).toBe('committed') + }) + expect(await Promise.all(files.map(async name => (await stat(join(f.directory, name))).ino))).toEqual(inodes) + expect(await f.control('snapshot_archive_owner_slots').whereNotNull('bindingJson')).toHaveLength(8) + } finally { + stop.resolve() + await Promise.allSettled(work) + } +}) + +test.each(['owner', 'slot', 'binding', 'request', 'released', 'state', 'expired'] as const)( + 'the final source check rejects a changed %s before reading wallet data', + async field => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + const guard = Backend.withSnapshotArchiveBackendGuard + jest.spyOn(Backend, 'withSnapshotArchiveBackendGuard').mockImplementationOnce(async (...args) => { + if (field === 'owner') await f.control('snapshot_archive_owners').delete() + else if (field === 'slot') await f.control('snapshot_archive_owners').update({ slot: 1 }) + else if (field === 'binding') + await f.control('snapshot_archive_owner_slots').where({ slot: 0 }).update({ bindingJson: '{}' }) + else if (field === 'request') await f.control('snapshot_archive_requests').delete() + else if (field === 'released') await f.control('snapshot_archive_requests').update({ released: 1 }) + else if (field === 'state') await f.control('snapshot_archive_requests').update({ state: 'ready' }) + else await f.control('snapshot_archive_requests').update({ expiresAt: 1 }) + return await guard(...args) + }) + const read = jest.fn() + await expect(readGuardedSnapshotArchive(f.control, f.open(), owner!, read)).rejects.toThrow('unavailable') + expect(read).not.toHaveBeenCalled() + } +) + +test.each(['invalid-slot', 'missing-slot'] as const)( + 'a corrupt %s cannot choose an unbound backend guard', + async kind => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + if (kind === 'invalid-slot') await f.control('snapshot_archive_owners').update({ slot: 8 }) + else await f.control('snapshot_archive_owner_slots').where({ slot: 0 }).delete() + await expect(Registry.readSnapshotArchiveOwnerGuard(f.control, owner!)).rejects.toThrow('unavailable') + } +) + +test('binding rechecks the owner slot and refuses to replace an established binding', async () => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + const context = await Registry.readSnapshotArchiveOwnerGuard(f.control, owner!) + const verify = jest.fn(async () => undefined) + await f.control('snapshot_archive_owners').update({ slot: 1 }) + await expect(Registry.bindSnapshotArchiveOwnerGuard(f.control, context, 'binding', verify)).rejects.toThrow( + 'unavailable' + ) + await f.control('snapshot_archive_owners').update({ slot: 0 }) + const bound = await Registry.bindSnapshotArchiveOwnerGuard(f.control, context, 'binding', verify) + expect(bound.bindingJson).toBe('binding') + await expect(Registry.bindSnapshotArchiveOwnerGuard(f.control, bound, 'replacement', verify)).rejects.toThrow( + 'unavailable' + ) + expect((await f.control('snapshot_archive_owner_slots').where({ slot: 0 }).first()).bindingJson).toBe('binding') +}) + +test('recovery fences only the exact expired owner and preserves live requests', async () => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + const context = await Registry.readSnapshotArchiveOwnerGuard(f.control, owner!) + const fence = (value = context, acknowledge = false) => + f.control.transaction(trx => Registry.fenceSnapshotArchiveOwnerGuard(trx, value, acknowledge)) + expect(await fence()).toBe(false) + expect(await fence({ ...context, slot: 1 })).toBe(false) + expect(await fence({ ...context, bindingJson: 'different' })).toBe(false) + await f.control('snapshot_archive_requests').where(owner!).update({ expiresAt: 1 }) + expect(await fence()).toBe(true) + expect((await f.control('snapshot_archive_requests').where(owner!).first()).state).toBe('expired') + expect(await f.control('snapshot_archive_owners')).toHaveLength(1) + expect(await fence(context, true)).toBe(true) + expect(await f.control('snapshot_archive_owners')).toHaveLength(0) + expect(await fence(context, true)).toBe(false) +}) + +test.each(['missing', 'released'] as const)('recovery refuses a %s request without releasing its owner', async kind => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + const context = await Registry.readSnapshotArchiveOwnerGuard(f.control, owner!) + if (kind === 'missing') await f.control('snapshot_archive_requests').delete() + else await f.control('snapshot_archive_requests').update({ released: 1 }) + await expect( + f.control.transaction(trx => Registry.fenceSnapshotArchiveOwnerGuard(trx, context, true)) + ).rejects.toThrow('unavailable') + expect(await f.control('snapshot_archive_owners')).toHaveLength(1) +}) + +test('a replaced guard marker or symlink cannot prove recovery of the bound source', async () => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + await readGuardedSnapshotArchive(f.control, f.open(), owner!, async () => undefined) + await f.requests.markCancellation(identity(), input) + const path = `${f.filename}.snapshot-owner-0.sqlite` + const metadata = knex({ ...f.config, connection: { filename: path } }) + try { + await metadata('snapshot_owner_guard').update({ marker: 'invalid' }) + } finally { + await metadata.destroy() + } + await expect(f.recover()).rejects.toThrow('identity changed') + await rename(path, path + '.retained') + await symlink(path + '.retained', path) + await expect(f.recover()).rejects.toThrow('identity changed') + expect(Number((await f.control('snapshot_archive_capacity').first()).archives)).toBe(1) +}) + +test('a marker changed after preparation is rejected on the guarded physical transaction', async () => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + const guard = Backend.withSnapshotArchiveBackendGuard + jest.spyOn(Backend, 'withSnapshotArchiveBackendGuard').mockImplementationOnce(async (...args) => { + const metadata = knex({ ...f.config, connection: { filename: `${f.filename}.snapshot-owner-0.sqlite` } }) + try { + await metadata('snapshot_owner_guard').update({ marker: 'a'.repeat(64) }) + } finally { + await metadata.destroy() + } + return await guard(...args) + }) + const read = jest.fn() + await expect(readGuardedSnapshotArchive(f.control, f.open(), owner!, read)).rejects.toThrow('identity changed') + expect(read).not.toHaveBeenCalled() +}) + +test('SQLite guards require WAL and reject a different physical database', async () => { + const f = await fixture() + await f.control.raw('PRAGMA journal_mode = DELETE') + await expect(Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null)).rejects.toThrow('identity changed') + await f.control.raw('PRAGMA journal_mode = WAL') + const binding = await Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null) + const other = await fixture() + await expect(Backend.assertSnapshotArchiveGuardBackend(other.control, binding)).rejects.toThrow('identity changed') +}) + +test('guard creation preserves an independent filesystem failure', async () => { + const f = await fixture() + const failure = Object.assign(new Error('fixture storage full'), { code: 'ENOSPC' }) + jest.spyOn(filesystem, 'open').mockRejectedValueOnce(failure) + await expect(Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null)).rejects.toBe(failure) +}) + +test('guard preparation rejects an inode replaced during metadata inspection', async () => { + const f = await fixture() + await Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null) + const path = `${f.filename}.snapshot-owner-0.sqlite` + const main = await filesystem.lstat(f.filename, { bigint: true }) + const before = await filesystem.lstat(path, { bigint: true }) + const after = await filesystem.lstat(path, { bigint: true }) + after.ino += 1n + jest.spyOn(filesystem, 'lstat').mockResolvedValueOnce(main).mockResolvedValueOnce(before).mockResolvedValueOnce(after) + await expect(Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null)).rejects.toThrow('identity changed') +}) + +test('a disconnected main database cannot select a guard from configuration alone', async () => { + const f = await fixture() + const raw = f.control.raw.bind(f.control) + const probe = { + client: f.control.client, + raw: (sql: string) => (sql === 'PRAGMA database_list' ? raw('SELECT 1 AS absent') : raw(sql)) + } as unknown as Knex + await expect(Backend.prepareSnapshotArchiveGuardBackend(probe, 0, null)).rejects.toThrow('identity changed') +}) + +test('physical connection rechecks both main and guard inode identity', async () => { + const f = await fixture() + const binding = await Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null) + const main = await filesystem.lstat(f.filename, { bigint: true }) + main.ino += 1n + const changedMain = jest.spyOn(filesystem, 'lstat').mockResolvedValueOnce(main) + await expect(Backend.assertSnapshotArchiveGuardBackend(f.control, binding)).rejects.toThrow('identity changed') + changedMain.mockRestore() + const mainActual = await filesystem.lstat(f.filename, { bigint: true }) + const guard = await filesystem.lstat(`${f.filename}.snapshot-owner-0.sqlite`, { bigint: true }) + guard.ino += 1n + jest.spyOn(filesystem, 'lstat').mockResolvedValueOnce(mainActual).mockResolvedValueOnce(guard) + await expect(Backend.assertSnapshotArchiveGuardBackend(f.control, binding)).rejects.toThrow('identity changed') +}) + +test('an absent owner cannot enter backend preparation', async () => { + const f = await fixture() + await expect( + Registry.readSnapshotArchiveOwnerGuard(f.control, { + identityKey: identity(), + requestId: 'a'.repeat(64), + claimToken: 'b'.repeat(64) + }) + ).rejects.toThrow('unavailable') +}) + +test('a concurrent exact acknowledgement makes an old recovery proof harmless', async () => { + const f = await fixture(), + input = request() + const { owner } = await f.requests.claim(identity(), input) + await readGuardedSnapshotArchive(f.control, f.open(), owner!, async () => undefined) + await f.requests.markCancellation(identity(), input) + const guard = Backend.withSnapshotArchiveBackendGuard + jest.spyOn(Backend, 'withSnapshotArchiveBackendGuard').mockImplementationOnce(async (...args) => { + await f.requests.sourceClosed(owner!) + return await guard(...args) + }) + await f.recover() + expect(await f.control('snapshot_archive_owners')).toHaveLength(0) + expect(Number((await f.control('snapshot_archive_capacity').first()).archives)).toBe(0) +}) + +test('omitting the guard flag preserves compatibility with the earlier owner schema', async () => { + const f = await fixture(false) + const owner = { identityKey: identity(), requestId: 'a'.repeat(64), claimToken: 'b'.repeat(64) } + await f.control.transaction(async trx => { + await ArchiveSql.lockSnapshotArchiveCapacity(trx) + await reserveSnapshotArchiveOwner(trx, owner) + }) + expect(await f.control('snapshot_archive_owners').first()).toEqual({ ...owner, slot: 0, archiveId: null }) + await addSnapshotArchiveGuardTable(f.control) + expect(await f.control('snapshot_archive_owners').first()).toMatchObject({ ...owner, guardVersion: 0 }) +}) + +test.each([-1, 8, Number.MAX_SAFE_INTEGER + 1])( + 'invalid owner slot %s is refused even if matching slot metadata exists', + async slot => { + const f = await fixture() + const { owner } = await f.requests.claim(identity(), request()) + await f.control('snapshot_archive_owners').where(owner!).update({ slot }) + await f.control('snapshot_archive_owner_slots').insert({ slot, bindingJson: null }) + await expect(Registry.readSnapshotArchiveOwnerGuard(f.control, owner!)).rejects.toThrow('unavailable') + } +) + +test.each([0.5, Number.NaN])('a driver returning noninteger owner slot %s is refused', async slot => { + const f = await fixture() + const { owner } = await f.requests.claim(identity(), request()) + f.control.client.config.postProcessResponse = (value: unknown) => + value !== null && typeof value === 'object' && Object.keys(value).length === 1 && 'slot' in value ? { slot } : value + await expect(Registry.readSnapshotArchiveOwnerGuard(f.control, owner!)).rejects.toThrow('unavailable') +}) + +test.each(['closed', 'failed', 'resource-limited'] as const)( + 'a %s owner is enumerated and fenced without rewriting its terminal outcome or acknowledging a live peer', + async state => { + const f = await fixture() + const { owner } = await f.requests.claim(identity(), request()) + const { owner: peer } = await f.requests.claim(identity(2), request(2)) + const context = await Registry.readSnapshotArchiveOwnerGuard(f.control, owner!) + await expect(Registry.assertSnapshotArchiveGuardOwner(f.control, { ...context, slot: 1 })).rejects.toThrow( + 'unavailable' + ) + await f.control('snapshot_archive_requests').where(owner!).update({ state }) + expect(await Registry.expiredSnapshotArchiveOwnerGuards(f.control)).toEqual([context]) + expect( + await f.control.transaction(trx => Registry.fenceSnapshotArchiveOwnerGuard(trx, { ...context, slot: 1 })) + ).toBe(false) + expect(await f.control.transaction(trx => Registry.fenceSnapshotArchiveOwnerGuard(trx, context))).toBe(true) + expect((await f.control('snapshot_archive_requests').where(owner!).first()).state).toBe(state) + expect(await f.control('snapshot_archive_owners')).toHaveLength(2) + expect(await f.control.transaction(trx => Registry.fenceSnapshotArchiveOwnerGuard(trx, context, true))).toBe(true) + expect(await f.control('snapshot_archive_owners')).toEqual([expect.objectContaining(peer!)]) + } +) + +test('the immutable deadline itself closes guard admission and permits expiry fencing', async () => { + const f = await fixture() + const input = request() + const { owner } = await f.requests.claim(identity(), input) + const context = await Registry.readSnapshotArchiveOwnerGuard(f.control, owner!) + jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(input.notAfter) + await expect(Registry.assertSnapshotArchiveGuardOwner(f.control, context)).rejects.toThrow('unavailable') + expect(await Registry.expiredSnapshotArchiveOwnerGuards(f.control)).toEqual([context]) + expect(await f.control.transaction(trx => Registry.fenceSnapshotArchiveOwnerGuard(trx, context))).toBe(true) + expect((await f.control('snapshot_archive_requests').where(owner!).first()).state).toBe('expired') + expect(await f.control('snapshot_archive_owners')).toHaveLength(1) +}) + +test.each(['g'.repeat(64), 'a'.repeat(63), 'a'.repeat(65), 'z' + 'a'.repeat(64), 'a'.repeat(64) + 'z'])( + 'an invalid initial guard marker %s cannot be accepted before the slot is bound', + async marker => { + const f = await fixture() + await Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null) + const metadata = knex({ ...f.config, connection: { filename: `${f.filename}.snapshot-owner-0.sqlite` } }) + try { + await metadata('snapshot_owner_guard').where({ id: 1 }).update({ marker }) + } finally { + await metadata.destroy() + } + await expect(Backend.prepareSnapshotArchiveGuardBackend(f.control, 0, null)).rejects.toThrow('identity changed') + expect((await f.control('snapshot_archive_owner_slots').where({ slot: 0 }).first()).bindingJson).toBeNull() + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuard.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuard.ts new file mode 100644 index 000000000..730c27332 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuard.ts @@ -0,0 +1,68 @@ +import { knex as createKnex, type Knex } from 'knex' +import { runInSeries } from '../../../utility/runInSeries' +import type { SnapshotArchiveRequestOwner } from './SnapshotArchiveRequest' +import { + assertSnapshotArchiveGuardBackend, + prepareSnapshotArchiveGuardBackend, + SnapshotArchiveGuardBusyError, + withSnapshotArchiveBackendGuard +} from './SnapshotArchiveGuardBackend' +import { + assertSnapshotArchiveGuardOwner, + bindSnapshotArchiveOwnerGuard, + expiredSnapshotArchiveOwnerGuards, + fenceSnapshotArchiveOwnerGuard, + readSnapshotArchiveOwnerGuard +} from './SnapshotArchiveGuardRegistry' + +/** Prepare outside the writer lock; bind and recheck the exact owner before taking its view. */ +export async function readGuardedSnapshotArchive( + control: Knex, + source: Knex, + owner: SnapshotArchiveRequestOwner, + read: (trx: Knex.Transaction) => Promise +): Promise { + const expected = await readSnapshotArchiveOwnerGuard(control, owner) + const binding = await prepareSnapshotArchiveGuardBackend(source, expected.slot, expected.bindingJson) + const context = await bindSnapshotArchiveOwnerGuard(control, expected, JSON.stringify(binding), trx => + assertSnapshotArchiveGuardBackend(trx, binding) + ) + return await withSnapshotArchiveBackendGuard(source, binding, async trx => { + await assertSnapshotArchiveGuardOwner(trx, context) + return await read(trx) + }) +} + +/** The provider serializes this operation; each proof pool closes before quota can be released. */ +export async function recoverSnapshotArchiveGuards(control: Knex, config: Knex.Config): Promise { + const owners = await expiredSnapshotArchiveOwnerGuards(control) + await runInSeries(owners, async context => { + if (context.bindingJson === null) { + // Recheck null under the capacity lock: a concurrent preparer may have + // bound the slot since enumeration. No guard may open before that bind. + await control.transaction(trx => fenceSnapshotArchiveOwnerGuard(trx, context, true)) + return + } + const proof = createKnex(config) + let fenced = false + try { + const binding = await prepareSnapshotArchiveGuardBackend(proof, context.slot, context.bindingJson) + fenced = await withSnapshotArchiveBackendGuard(proof, binding, async () => { + return await control.transaction(async trx => { + await assertSnapshotArchiveGuardBackend(trx, binding) + return await fenceSnapshotArchiveOwnerGuard(trx, context) + }) + }) + if (fenced) { + await control.transaction(async trx => { + await assertSnapshotArchiveGuardBackend(trx, binding) + await fenceSnapshotArchiveOwnerGuard(trx, context, true) + }) + } + } catch (error) { + if (!(error instanceof SnapshotArchiveGuardBusyError)) throw error + } finally { + await proof.destroy() + } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardBackend.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardBackend.test.ts new file mode 100644 index 000000000..79411ea5a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardBackend.test.ts @@ -0,0 +1,277 @@ +import { EventEmitter } from 'node:events' +import type { Knex } from 'knex' +import { SnapshotArchiveSourceCleanupError } from './KnexSnapshotArchiveSource' +import { + assertSnapshotArchiveGuardBackend, + prepareSnapshotArchiveGuardBackend, + SnapshotArchiveGuardBusyError, + withSnapshotArchiveBackendGuard, + type SnapshotArchiveGuardBinding +} from './SnapshotArchiveGuardBackend' + +const uuid = '12345678-1234-1234-1234-123456789abc' +const identity = { serverUuid: uuid, databaseName: 'fixture' } +function fixture() { + const stream = Object.assign(new EventEmitter(), { destroyed: false, closed: false }) + const connection = { stream } + const close = () => { + stream.destroyed = true + stream.closed = true + stream.emit('close') + } + const reply = jest.fn((sql: string): unknown => { + if (sql.startsWith('SELECT @@')) return [[identity]] + if (sql.startsWith('SELECT GET_LOCK')) return [[{ acquired: 1 }]] + return [] + }) + const queries: Array<{ sql: string; values: unknown; connection?: unknown }> = [] + const raw = jest.fn((sql: string, values?: unknown) => { + const entry = { sql, values, connection: undefined as unknown } + queries.push(entry) + return Object.assign( + Promise.resolve().then(() => reply(sql)), + { + connection(value: unknown) { + entry.connection = value + return this + } + } + ) + }) + const trx = { executionPromise: Promise.resolve(), rollback: jest.fn(async () => undefined) } + const destroy = jest.fn(async () => close()) + const client = { + config: { client: 'mysql2' }, + acquireConnection: jest.fn(async () => connection), + releaseConnection: jest.fn(async () => undefined) + } + const transaction = jest.fn(async () => trx) + const k = { raw, client, transaction, destroy } as unknown as Knex + return { k, stream, close, connection, reply, queries, trx, destroy, client, transaction } +} + +test('MySQL guard namespaces bind the actual server, database and bounded slot', async () => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + expect(binding).toMatchObject({ version: 1, kind: 'mysql', serverUuid: uuid, database: 'fixture' }) + if (binding.kind !== 'mysql') throw new Error('Expected MySQL binding') + expect(binding.lock).toMatch(/^wallet-snapshot-v1:[0-9a-f]{40}:0$/) + expect(binding.lock.length).toBeLessThanOrEqual(64) + expect(await prepareSnapshotArchiveGuardBackend(f.k, 0, JSON.stringify(binding))).toEqual(binding) + expect(await prepareSnapshotArchiveGuardBackend(f.k, 7, null)).not.toEqual(binding) + for (const slot of [-1, 8, 0.5, Number.NaN, Number.MAX_SAFE_INTEGER + 1]) + await expect(prepareSnapshotArchiveGuardBackend(f.k, slot, null)).rejects.toThrow('identity changed') + await expect(prepareSnapshotArchiveGuardBackend(f.k, 1, JSON.stringify(binding))).rejects.toThrow('identity changed') +}) + +test.each([ + undefined, + { serverUuid: null, databaseName: 'fixture' }, + { serverUuid: 'not-a-server', databaseName: 'fixture' }, + { serverUuid: 'a' + uuid, databaseName: 'fixture' }, + { serverUuid: uuid + 'a', databaseName: 'fixture' }, + { serverUuid: uuid, databaseName: undefined }, + { serverUuid: uuid, databaseName: '' }, + { serverUuid: uuid, databaseName: 'x'.repeat(65) } +])('invalid MySQL identity %j refuses a guard', async row => { + const f = fixture() + f.reply.mockReturnValue([[row]]) + await expect(prepareSnapshotArchiveGuardBackend(f.k, 0, null)).rejects.toThrow('identity changed') +}) + +test.each(['server', 'database'] as const)( + 'physical %s mismatch refuses reads and closes its private connection', + async field => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + f.reply.mockImplementation(sql => + sql.startsWith('SELECT @@') + ? [[field === 'server' ? { ...identity, serverUuid: 'a'.repeat(36) } : { ...identity, databaseName: 'other' }]] + : [[{ acquired: 1 }]] + ) + const read = jest.fn() + await expect(withSnapshotArchiveBackendGuard(f.k, binding, read)).rejects.toThrow('identity changed') + expect(read).not.toHaveBeenCalled() + expect(f.transaction).not.toHaveBeenCalled() + expect(f.queries.at(-1)?.connection).toBe(f.connection) + expect(f.stream.closed).toBe(true) + expect(f.client.releaseConnection).toHaveBeenCalledWith(f.connection) + } +) + +test.each([0, null, undefined, 2])('MySQL GET_LOCK result %s never admits a read', async acquired => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + f.reply.mockImplementation(sql => (sql.startsWith('SELECT @@') ? [[identity]] : [[{ acquired }]])) + const read = jest.fn() + const work = withSnapshotArchiveBackendGuard(f.k, binding, read) + if (acquired === 0) await expect(work).rejects.toBeInstanceOf(SnapshotArchiveGuardBusyError) + else await expect(work).rejects.toThrow('identity changed') + expect(read).not.toHaveBeenCalled() + expect(f.transaction).not.toHaveBeenCalled() + expect(f.stream.closed).toBe(true) +}) + +test('a graceful quit and a destroyed socket still wait for physical close before rollback or success', async () => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + let quit!: () => void + const quitting = new Promise(resolve => { + quit = resolve + }) + f.destroy.mockImplementation(async () => { + f.stream.destroyed = true + quit() + }) + let settled = false + const work = withSnapshotArchiveBackendGuard(f.k, binding, async trx => { + expect(trx).toBe(f.trx) + return 42 + }).finally(() => { + settled = true + }) + void work.catch(() => undefined) + try { + await quitting + await new Promise(resolve => setImmediate(resolve)) + expect(settled).toBe(false) + expect(f.trx.rollback).not.toHaveBeenCalled() + expect(f.stream.listenerCount('close')).toBe(1) + expect(f.queries.slice(1).map(query => query.sql)).toEqual([ + 'SELECT @@server_uuid AS serverUuid, DATABASE() AS databaseName', + 'SELECT GET_LOCK(?, 0) AS acquired', + 'SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY' + ]) + expect(f.queries.slice(1).every(query => query.connection === f.connection)).toBe(true) + expect(f.queries[2].values).toEqual([(binding as Extract).lock]) + expect(f.transaction).toHaveBeenCalledWith({ connection: f.connection }) + f.close() + await expect(work).resolves.toBe(42) + expect(f.trx.rollback).toHaveBeenCalledTimes(1) + expect(f.stream.listenerCount('close')).toBe(0) + } finally { + if (!f.stream.closed) f.close() + await work.catch(() => undefined) + } +}) + +test('a socket destroyed before cleanup still waits for its pending close event', async () => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + let cleanup!: () => void + const started = new Promise(resolve => { + cleanup = resolve + }) + f.destroy.mockImplementation(async () => { + cleanup() + }) + const work = withSnapshotArchiveBackendGuard(f.k, binding, async () => { + f.stream.destroyed = true + return 'read' + }) + void work.catch(() => undefined) + try { + await started + expect(f.stream.listenerCount('close')).toBe(1) + expect(f.trx.rollback).not.toHaveBeenCalled() + f.close() + await expect(work).resolves.toBe('read') + } finally { + if (!f.stream.closed) f.close() + await work.catch(() => undefined) + } +}) + +test.each([new Error('read failed'), undefined])( + 'proved close preserves the exact independent read failure %s', + async failure => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + f.trx.rollback.mockRejectedValue(new Error('connection already closed')) + await expect( + withSnapshotArchiveBackendGuard(f.k, binding, async () => { + throw failure + }) + ).rejects.toBe(failure) + expect(f.stream.closed).toBe(true) + expect(f.stream.listenerCount('close')).toBe(0) + } +) + +test.each(['destroy', 'release'] as const)( + 'a %s failure retains the manual transaction and reports cleanup failure', + async phase => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + const failure = new Error('native cleanup failed') + f.destroy.mockImplementation(async () => undefined) + if (phase === 'destroy') f.destroy.mockRejectedValue(failure) + else f.client.releaseConnection.mockRejectedValue(failure) + try { + await expect(withSnapshotArchiveBackendGuard(f.k, binding, async () => 1)).rejects.toBeInstanceOf( + SnapshotArchiveSourceCleanupError + ) + expect(f.trx.rollback).not.toHaveBeenCalled() + expect(f.stream.closed).toBe(false) + expect(f.stream.listenerCount('close')).toBe(0) + } finally { + f.close() + } + } +) + +test('a previously closed socket does not need another close event', async () => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + f.destroy.mockImplementation(async () => undefined) + await expect( + withSnapshotArchiveBackendGuard(f.k, binding, async () => { + f.close() + return 3 + }) + ).resolves.toBe(3) + expect(f.stream.listenerCount('close')).toBe(0) + await assertSnapshotArchiveGuardBackend(f.k, binding) + expect(f.queries.at(-1)?.connection).toBeUndefined() +}) + +test('unsupported backend refuses before acquiring a source', async () => { + const f = fixture() + f.client.config.client = 'other' + await expect(prepareSnapshotArchiveGuardBackend(f.k, 0, null)).rejects.toThrow('require better-sqlite3 WAL or MySQL') + expect(f.client.acquireConnection).not.toHaveBeenCalled() +}) + +test.each([1, 64])('MySQL accepts a valid database name of exactly %s characters', async length => { + const f = fixture() + const databaseName = 'd'.repeat(length) + f.reply.mockReturnValue([[{ serverUuid: uuid.toUpperCase(), databaseName }]]) + expect(await prepareSnapshotArchiveGuardBackend(f.k, 0, null)).toMatchObject({ + kind: 'mysql', + serverUuid: uuid.toUpperCase(), + database: databaseName + }) +}) + +test.each(['missing-stream', 'unproved-event'] as const)( + 'MySQL cleanup refuses %s without treating the read transaction as released', + async kind => { + const f = fixture() + const binding = await prepareSnapshotArchiveGuardBackend(f.k, 0, null) + if (kind === 'missing-stream') Reflect.deleteProperty(f.connection, 'stream') + else + f.destroy.mockImplementation(async () => { + f.stream.emit('close') + }) + try { + await expect(withSnapshotArchiveBackendGuard(f.k, binding, async () => 1)).rejects.toMatchObject({ + name: 'SnapshotArchiveSourceCleanupError', + cause: { message: 'Snapshot archive source connection did not close' } + }) + expect(f.trx.rollback).not.toHaveBeenCalled() + expect(f.stream.listenerCount('close')).toBe(0) + } finally { + f.close() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardBackend.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardBackend.ts new file mode 100644 index 000000000..97d88fc37 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardBackend.ts @@ -0,0 +1,254 @@ +import { createHash, randomBytes } from 'node:crypto' +import { lstat, open, realpath } from 'node:fs/promises' +import type { Duplex } from 'node:stream' +import { knex as createKnex, type Knex } from 'knex' +import { WERR_INVALID_OPERATION, WERR_NOT_IMPLEMENTED } from '../../../sdk/WERR_errors' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { SnapshotArchiveSourceCleanupError } from './KnexSnapshotArchiveSource' +import { snapshotArchiveLimits } from './SnapshotArchive' + +interface FileIdentity { + path: string + device: string + inode: string +} + +interface SQLiteGuardBinding { + version: 1 + kind: 'sqlite' + database: FileIdentity + guard: FileIdentity & { marker: string } +} + +interface MySQLGuardBinding { + version: 1 + kind: 'mysql' + serverUuid: string + database: string + lock: string +} + +export type SnapshotArchiveGuardBinding = SQLiteGuardBinding | MySQLGuardBinding + +export class SnapshotArchiveGuardBusyError extends SnapshotResourceLimitError { + constructor() { + super('Snapshot archive source guard is occupied') + } +} + +function unavailable(): never { + throw new WERR_INVALID_OPERATION('Snapshot archive source backend or guard identity changed') +} + +async function fileIdentity(path: string): Promise { + const info = await lstat(path, { bigint: true }) + if (!info.isFile()) unavailable() + return { path, device: info.dev.toString(), inode: info.ino.toString() } +} + +function sqliteGuardPool(filename: string, existing: boolean): Knex { + // Knex's option declaration omits this supported better-sqlite3 option. + const options = { fileMustExist: true, readonly: existing } + return createKnex({ + client: 'better-sqlite3', + connection: { filename, options }, + useNullAsDefault: true, + pool: { min: 0, max: 1 }, + acquireConnectionTimeout: 5000 + }) +} + +async function guardFile(database: FileIdentity, slot: number, existing: boolean): Promise { + const path = `${database.path}.snapshot-owner-${slot}.sqlite` + if (!existing) { + try { + const file = await open(path, 'wx', 0o600) + await file.close() + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error + } + } + const before = await fileIdentity(path) + const guard = sqliteGuardPool(path, existing) + try { + if (!existing) { + await guard.raw('PRAGMA busy_timeout = 0') + await guard.transaction(async trx => { + await trx.raw( + 'CREATE TABLE IF NOT EXISTS snapshot_owner_guard (id INTEGER PRIMARY KEY, marker TEXT NOT NULL, held INTEGER NOT NULL)' + ) + await trx('snapshot_owner_guard') + .insert({ id: 1, marker: randomBytes(32).toString('hex'), held: 0 }) + .onConflict('id') + .ignore() + }) + } + const row: { marker?: unknown } | undefined = await guard('snapshot_owner_guard').where({ id: 1 }).first('marker') + if (typeof row?.marker !== 'string' || !/^[0-9a-f]{64}$/.test(row.marker)) unavailable() + if (JSON.stringify(await fileIdentity(path)) !== JSON.stringify(before)) unavailable() + return { version: 1, kind: 'sqlite', database, guard: { ...before, marker: row.marker } } + } finally { + await guard.destroy() + } +} + +async function mysqlIdentity(k: Knex, connection?: unknown): Promise<{ serverUuid: string; database: string }> { + const query = k.raw('SELECT @@server_uuid AS serverUuid, DATABASE() AS databaseName') + if (connection !== undefined) void query.connection(connection) + const [rows]: Array> = await query + const row = rows[0] + if ( + typeof row?.serverUuid !== 'string' || + !/^[0-9a-f-]{36}$/i.test(row.serverUuid) || + typeof row.databaseName !== 'string' || + row.databaseName.length < 1 || + row.databaseName.length > 64 + ) + unavailable() + return { serverUuid: row.serverUuid, database: row.databaseName } +} + +/** No main writer lock is held while probing files or opening a guard metadata connection. */ +export async function prepareSnapshotArchiveGuardBackend( + k: Knex, + slot: number, + existing: string | null +): Promise { + if (!Number.isSafeInteger(slot) || slot < 0 || slot >= snapshotArchiveLimits.archives) unavailable() + let binding: SnapshotArchiveGuardBinding + if (String(k.client.config.client).includes('mysql')) { + const identity = await mysqlIdentity(k) + const namespace = createHash('sha256').update(JSON.stringify(identity)).digest('hex').slice(0, 40) + binding = { version: 1, kind: 'mysql', ...identity, lock: `wallet-snapshot-v1:${namespace}:${slot}` } + } else if (k.client.config.client === 'better-sqlite3') { + const modes: Array<{ journal_mode: string }> = await k.raw('PRAGMA journal_mode') + if (modes[0]?.journal_mode.toLowerCase() !== 'wal') unavailable() + const rows: Array<{ name: string; file: string }> = await k.raw('PRAGMA database_list') + const filename = rows.find(row => row.name === 'main')?.file + if (typeof filename !== 'string' || filename.length === 0) unavailable() + const database = await fileIdentity(await realpath(filename)) + binding = await guardFile(database, slot, existing !== null) + } else { + throw new WERR_NOT_IMPLEMENTED('Snapshot archive source guards require better-sqlite3 WAL or MySQL') + } + if (existing !== null && JSON.stringify(binding) !== existing) unavailable() + return binding +} + +/** Recheck the physical connection, not merely the configured address or an earlier pooled query. */ +export async function assertSnapshotArchiveGuardBackend( + k: Knex, + binding: SnapshotArchiveGuardBinding, + connection?: unknown +): Promise { + if (binding.kind === 'mysql') { + const actual = await mysqlIdentity(k, connection) + if (actual.serverUuid !== binding.serverUuid || actual.database !== binding.database) unavailable() + } else { + const query = k.raw('PRAGMA database_list') + if (connection !== undefined) void query.connection(connection) + const rows: Array<{ name: string; file: string }> = await query + const filename = rows.find(row => row.name === 'main')?.file + if (typeof filename !== 'string' || (await realpath(filename)) !== binding.database.path) unavailable() + if (JSON.stringify(await fileIdentity(binding.database.path)) !== JSON.stringify(binding.database)) unavailable() + const { marker: _marker, ...guard } = binding.guard + if (JSON.stringify(await fileIdentity(guard.path)) !== JSON.stringify(guard)) unavailable() + } +} + +async function attachGuard(k: Knex, connection: unknown, binding: SnapshotArchiveGuardBinding): Promise { + await assertSnapshotArchiveGuardBackend(k, binding, connection) + if (binding.kind === 'mysql') { + const [rows]: Array> = await k + .raw('SELECT GET_LOCK(?, 0) AS acquired', [binding.lock]) + .connection(connection) + if (rows[0]?.acquired === 0) throw new SnapshotArchiveGuardBusyError() + if (rows[0]?.acquired !== 1) unavailable() + await k.raw('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY').connection(connection) + } else { + await k.raw('ATTACH DATABASE ? AS snapshot_owner', [binding.guard.path]).connection(connection) + await k.raw('PRAGMA busy_timeout = 0').connection(connection) + } +} + +async function holdSQLiteGuard(trx: Knex, binding: SQLiteGuardBinding): Promise { + try { + const changed = await trx('snapshot_owner.snapshot_owner_guard') + .where({ id: 1, marker: binding.guard.marker }) + .update({ held: 1 }) + if (changed !== 1) unavailable() + // The private connection only reads the wallet after establishing its guard. + await trx.raw('PRAGMA query_only = ON') + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'SQLITE_BUSY') throw new SnapshotArchiveGuardBusyError() + throw error + } +} + +/** This pool is exclusively owned by this operation; no connection is returned for reuse. */ +async function closeGuardPool(k: Knex, connection: { open?: boolean; stream?: Duplex }): Promise { + const stream = connection.stream + let closedListener: (() => void) | undefined + const nativeClose = + stream === undefined || stream.closed + ? undefined + : new Promise(resolve => { + closedListener = resolve + stream.once('close', resolve) + }) + try { + const destruction = k.destroy() + const settled = await Promise.allSettled([destruction, k.client.releaseConnection(connection)]) + for (const result of settled) if (result.status === 'rejected') throw result.reason + // mysql2's graceful-quit callback precedes the socket's physical close. + await nativeClose + const closed = String(k.client.config.client).includes('mysql') + ? connection.stream?.closed === true + : connection.open === false + if (closed !== true) throw new Error('Snapshot archive source connection did not close') + } catch (error) { + throw new SnapshotArchiveSourceCleanupError(error) + } finally { + if (closedListener !== undefined) stream!.removeListener('close', closedListener) + } +} + +/** + * Keep the same-connection guard until physical destruction. SQLite's normal + * COMMIT would release it too early, so the private pool closes the still-open + * read transaction. Knex then settles its expected closed-connection rollback; + * only proved physical closure can supersede that internal completion result. + */ +export async function withSnapshotArchiveBackendGuard( + k: Knex, + binding: SnapshotArchiveGuardBinding, + read: (trx: Knex.Transaction) => Promise +): Promise { + const connection = await k.client.acquireConnection() + let released = false + const close = async (): Promise => { + released = true + await closeGuardPool(k, connection) + } + try { + await attachGuard(k, connection, binding) + const trx = await k.transaction({ connection }) + void trx.executionPromise.catch(() => undefined) + let outcome: { ok: true; value: T } | { ok: false; error: unknown } + try { + if (binding.kind === 'sqlite') await holdSQLiteGuard(trx, binding) + outcome = { ok: true, value: await read(trx) } + } catch (error) { + outcome = { ok: false, error } + } + // A failed physical close deliberately leaves this manual transaction and + // its guard held. Automatic rollback would falsely prove source cleanup. + await close() + await trx.rollback().catch(() => undefined) + await trx.executionPromise.catch(() => undefined) + if (!outcome.ok) throw outcome.error + return outcome.value + } finally { + if (!released) await close() + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardRegistry.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardRegistry.ts new file mode 100644 index 000000000..1bdf57509 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveGuardRegistry.ts @@ -0,0 +1,129 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { snapshotArchiveLimits } from './SnapshotArchive' +import type { SnapshotArchiveRequestOwner } from './SnapshotArchiveRequest' +import { lockSnapshotArchiveCapacity, snapshotArchiveDatabaseNow } from './SnapshotArchiveSql' + +export interface SnapshotArchiveOwnerGuard { + owner: SnapshotArchiveRequestOwner + slot: number + bindingJson: string | null +} + +interface RequestState { + state: string + expiresAt: number | string + released: number | boolean +} + +const terminal = ['closed', 'failed', 'expired', 'resource-limited'] + +function unavailable(): never { + throw new WERR_INVALID_OPERATION('Snapshot archive source guard is unavailable') +} + +async function ownerGuard(k: Knex, owner: SnapshotArchiveRequestOwner): Promise { + const row: { slot: number } | undefined = await k('snapshot_archive_owners') + .where({ ...owner, guardVersion: 1 }) + .first('slot') + if (row === undefined) return undefined + if (!Number.isSafeInteger(row.slot) || row.slot < 0 || row.slot >= snapshotArchiveLimits.archives) unavailable() + const slot: { bindingJson: string | null } | undefined = await k('snapshot_archive_owner_slots') + .where({ slot: row.slot }) + .first('bindingJson') + if (slot === undefined || (slot.bindingJson !== null && typeof slot.bindingJson !== 'string')) unavailable() + return { owner: { ...owner }, slot: row.slot, bindingJson: slot.bindingJson } +} + +/** The source calls this only after acquiring the backend guard, before any wallet-data read. */ +export async function assertSnapshotArchiveGuardOwner(k: Knex, expected: SnapshotArchiveOwnerGuard): Promise { + const actual = await ownerGuard(k, expected.owner) + const request: RequestState | undefined = await k('snapshot_archive_requests').where(expected.owner).first() + if ( + actual?.slot !== expected.slot || + actual.bindingJson !== expected.bindingJson || + request === undefined || + request.released || + !['claimed', 'capturing'].includes(request.state) || + Number(request.expiresAt) <= (await snapshotArchiveDatabaseNow(k)) + ) + unavailable() +} + +/** Leave the main writer lock before filesystem or backend preparation. */ +export async function readSnapshotArchiveOwnerGuard( + knex: Knex, + input: SnapshotArchiveRequestOwner +): Promise { + const owner = { ...input } + return await knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + const context = await ownerGuard(trx, owner) + if (context === undefined) unavailable() + await assertSnapshotArchiveGuardOwner(trx, context) + return context + }) +} + +/** Fixed slot bindings survive owner release; delayed claimants cannot select a new guard. */ +export async function bindSnapshotArchiveOwnerGuard( + knex: Knex, + context: SnapshotArchiveOwnerGuard, + bindingJson: string, + verifyBackend: (trx: Knex) => Promise +): Promise { + return await knex.transaction(async trx => { + await lockSnapshotArchiveCapacity(trx) + await verifyBackend(trx) + const current = await ownerGuard(trx, context.owner) + if (current?.slot !== context.slot) unavailable() + await assertSnapshotArchiveGuardOwner(trx, current) + if (current.bindingJson !== null && current.bindingJson !== bindingJson) unavailable() + await trx('snapshot_archive_owner_slots').where({ slot: current.slot }).update({ bindingJson }) + return { ...current, bindingJson } + }) +} + +/** At most eight owners; older unguarded candidates remain reserved for explicit cleanup. */ +export async function expiredSnapshotArchiveOwnerGuards(knex: Knex): Promise { + const now = await snapshotArchiveDatabaseNow(knex) + const rows: Array = await knex( + 'snapshot_archive_owners as owner' + ) + .join('snapshot_archive_requests as request', function () { + this.on('owner.identityKey', 'request.identityKey') + .andOn('owner.requestId', 'request.requestId') + .andOn('owner.claimToken', 'request.claimToken') + }) + .join('snapshot_archive_owner_slots as slot', 'slot.slot', 'owner.slot') + .where('owner.guardVersion', 1) + .where(query => { + void query.where('request.expiresAt', '<=', now).orWhereIn('request.state', terminal) + }) + .select('owner.identityKey', 'owner.requestId', 'owner.claimToken', 'owner.slot', 'slot.bindingJson') + .limit(snapshotArchiveLimits.archives) + return rows.map(({ slot, bindingJson, ...owner }) => ({ owner, slot, bindingJson })) +} + +/** Caller holds the backend guard, or proves the binding is still null under this lock. */ +export async function fenceSnapshotArchiveOwnerGuard( + k: Knex, + context: SnapshotArchiveOwnerGuard, + acknowledge = false +): Promise { + await lockSnapshotArchiveCapacity(k) + const actual = await ownerGuard(k, context.owner) + if (actual?.slot !== context.slot || actual.bindingJson !== context.bindingJson) return false + const request: RequestState | undefined = await k('snapshot_archive_requests').where(context.owner).first() + if (request === undefined || request.released) unavailable() + const expired = Number(request.expiresAt) <= (await snapshotArchiveDatabaseNow(k)) + if (!expired && !terminal.includes(request.state)) return false + if (!terminal.includes(request.state)) { + await k('snapshot_archive_requests').where(context.owner).update({ state: 'expired' }) + } + if (acknowledge) + await k('snapshot_archive_owners') + .where({ ...context.owner, slot: context.slot }) + .delete() + return true +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveHttp.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveHttp.test.ts new file mode 100644 index 000000000..8eb26e5a2 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveHttp.test.ts @@ -0,0 +1,213 @@ +import { PrivateKey, ProtoWallet } from '@bsv/sdk' +import { StorageClient } from '../../remoting/StorageClient' +import { StorageClient as StorageMobile } from '../../remoting/StorageMobile' +import { KnexSnapshotArchiveService } from './KnexSnapshotArchiveService' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import { snapshotArchiveCapabilities } from './SnapshotArchiveProtocol' +import { snapshotHttpFixture, gate } from '../../../../test/utils/snapshotArchiveHttpFixtures' +import { decodeSyncTransfer } from '../../remoting/SyncTransfer' + +afterEach(() => jest.restoreAllMocks()) + +test.each([StorageClient, StorageMobile])( + 'authenticated %p capture survives lost admission and server replacement without recapture', + async Client => { + const fixture = await snapshotHttpFixture() + const allowOpen = gate() + try { + const { storage, identityKey, wallet, serve } = fixture + let { server, url } = await serve() + let client = new Client(wallet, url, { + serverIdentityKey: fixture.serverIdentityKey, + storageIdentityKey: 'http-snapshot-source' + }) + const settings = await client.makeAvailable() + const ordinaryRpc = Reflect.get(client, 'rpcCall').bind(client) + await expect(ordinaryRpc('releaseSyncTransfer', [{ identityKey, transferId: 'e'.repeat(64) }])).rejects.toThrow( + 'Wallet sync transfer expired' + ) + await expect( + ordinaryRpc('readSyncTransferPart', [{ identityKey, transferId: 'e'.repeat(64), offset: 0 }]) + ).rejects.toThrow('Wallet sync transfer expired') + expect(settings.snapshotArchive).toEqual(snapshotArchiveCapabilities) + expect(storage.getSettings()).not.toHaveProperty('snapshotArchive') + let transport = (await client.getSnapshotArchiveTransport(identityKey))! + const offer = await transport.offer() + expect(offer.sourceSchema).toBe('2026-10-02-001 repair snapshot SQLite conflict maintenance') + expect(Math.abs(offer.serverTime - Date.now())).toBeLessThan(5000) + const fields = { + version: 1 as const, + nonce: 'a'.repeat(64), + notAfter: offer.serverTime + 300000, + maxBytes: 32768 + } + const request = { ...fields, requestId: snapshotArchiveRequestId(fields) } + const entered = gate() + const original = storage.openSnapshotArchiveSource.bind(storage) + const opening = jest.spyOn(storage, 'openSnapshotArchiveSource').mockImplementation(async (...args) => { + entered.resolve() + await allowOpen.promise + return await original(...args) + }) + const rpc = Reflect.get(client, 'snapshotRpcCall').bind(client) + const lost = jest.spyOn(client as never, 'snapshotRpcCall' as never).mockImplementation((async ( + method: string, + params: unknown[], + signal?: AbortSignal + ) => { + const response = await rpc(method, params, signal) + if (method === 'startSnapshotArchive') throw new Error('synthetic lost admission acknowledgement') + return response + }) as never) + await expect(transport.start(request)).rejects.toThrow('lost admission') + await entered.promise + lost.mockRestore() + expect((await transport.status(request)).state).toBe('building') + expect((await transport.start(request)).state).toBe('building') + expect(opening).toHaveBeenCalledTimes(1) + // A foreground write is possible while the owned reader is opening. + await storage.knex('tx_labels').where({ txLabelId: 1 }).update({ label: 'foreground' }) + const controller = Reflect.get(Reflect.get(server, 'snapshotArchives'), 'service') as KnexSnapshotArchiveService + allowOpen.resolve() + const ready = await controller.create(identityKey, request) + expect(await transport.status(request)).toEqual(ready) + const verified = await transport.directory(ready, offer, Date.now()) + expect(verified.manifest.pages).toBe(13) + const page = await transport.page(verified, verified.tables.txLabels.first) + const decoded = decodeSyncTransfer(page) as { table: string; rows: unknown[] } + expect(decoded.table).toBe('txLabels') + expect(decoded.rows).toEqual([ + expect.objectContaining({ txLabelId: 1, label: 'foreground', isDeleted: false }), + expect.objectContaining({ txLabelId: 3, isDeleted: true }) + ]) + expect(JSON.stringify(ready)).not.toMatch(/claimToken|writerToken/) + expect(JSON.stringify(verified)).not.toMatch(/claimToken|writerToken/) + + const strangerKey = PrivateKey.fromRandom() + const stranger = new Client(new ProtoWallet(strangerKey), url) + const foreignTransport = (await stranger.getSnapshotArchiveTransport(identityKey))! + await expect(foreignTransport.offer()).rejects.toThrow('match authentication') + await expect(foreignTransport.start(request)).rejects.toThrow('match authentication') + await expect(foreignTransport.status(request)).rejects.toThrow('match authentication') + await expect(foreignTransport.cancel(request.requestId)).rejects.toThrow('match authentication') + const ownForeign = (await stranger.getSnapshotArchiveTransport(strangerKey.toPublicKey().toString()))! + await expect(ownForeign.status(request)).rejects.toThrow('unavailable') + await expect(ownForeign.page(verified, 0)).rejects.toThrow('unavailable') + await ownForeign.cancel(request.requestId) + expect(await transport.status(request)).toEqual(ready) + + await server.close() + ;({ server, url } = await serve()) + client = new Client(wallet, url, { serverIdentityKey: fixture.serverIdentityKey }) + transport = (await client.getSnapshotArchiveTransport(identityKey))! + expect(await transport.start(request)).toEqual(ready) + expect(opening).toHaveBeenCalledTimes(1) + expect( + await transport.page(await transport.directory(ready, offer, Date.now()), verified.tables.txLabels.first) + ).toEqual(page) + await transport.cancel(request.requestId) + expect((await transport.start(request)).state).toBe('closed') + await expect(transport.page(verified, 0)).rejects.toThrow('unavailable') + expect(await storage.knex('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + } finally { + allowOpen.resolve() + await fixture.close() + } + } +) + +test.each(['server-disabled', 'provider-disabled', 'small-response', 'old-schema'] as const)( + 'does not advertise snapshot transport for %s and legacy RPC still works', + async variant => { + const fixture = await snapshotHttpFixture(variant !== 'provider-disabled') + try { + if (variant === 'old-schema') await fixture.storage.knex.schema.dropTable('snapshot_archive_requests') + const { url } = await fixture.serve({ + snapshotArchives: variant !== 'server-disabled', + ...(variant === 'small-response' ? { maxRpcResponseBytes: 8192 } : {}) + }) + const client = new StorageClient(fixture.wallet, url) + expect((await client.makeAvailable()).snapshotArchive).toBeUndefined() + expect(await client.getSnapshotArchiveTransport(fixture.identityKey)).toBeUndefined() + expect((await client.findOrInsertUser(fixture.identityKey)).user.identityKey).toBe(fixture.identityKey) + const rpc = Reflect.get(client, 'rpcCall').bind(client) + await expect(rpc('getSnapshotArchiveOffer', [{ version: 1, identityKey: fixture.identityKey }])).rejects.toThrow( + variant === 'server-disabled' || variant === 'small-response' ? 'network error 400' : 'unavailable' + ) + } finally { + await fixture.close() + } + } +) + +test('client rollback disables snapshots; small capture budget has a durable distinct resource-limit receipt', async () => { + const fixture = await snapshotHttpFixture() + try { + const { server, url } = await fixture.serve() + const disabled = new StorageMobile(fixture.wallet, url, { snapshotArchives: false }) + expect(await disabled.getSnapshotArchiveTransport(fixture.identityKey)).toBeUndefined() + const client = new StorageClient(fixture.wallet, url) + const transport = (await client.getSnapshotArchiveTransport(fixture.identityKey))! + const offer = await transport.offer() + const fields = { version: 1 as const, nonce: 'b'.repeat(64), notAfter: offer.serverTime + 300000, maxBytes: 4097 } + const request = { ...fields, requestId: snapshotArchiveRequestId(fields) } + expect((await transport.start(request)).state).toBe('building') + const service = Reflect.get(Reflect.get(server, 'snapshotArchives'), 'service') as KnexSnapshotArchiveService + await service.create(fixture.identityKey, request).catch(() => undefined) + expect((await transport.status(request)).state).toBe('resource-limited') + expect((await transport.start(request)).state).toBe('resource-limited') + expect(await fixture.storage.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 0, + reservedBytes: 0 + }) + } finally { + await fixture.close() + } +}) + +test('server close fences admission and awaits opening capture, physical pool cleanup and HTTP callback', async () => { + const fixture = await snapshotHttpFixture() + const allowOpen = gate() + try { + const { server, url } = await fixture.serve() + const client = new StorageClient(fixture.wallet, url) + const transport = (await client.getSnapshotArchiveTransport(fixture.identityKey))! + const offer = await transport.offer() + const fields = { version: 1 as const, nonce: 'c'.repeat(64), notAfter: offer.serverTime + 300000, maxBytes: 32768 } + const request = { ...fields, requestId: snapshotArchiveRequestId(fields) } + const entered = gate() + const original = fixture.storage.openSnapshotArchiveSource.bind(fixture.storage) + jest.spyOn(fixture.storage, 'openSnapshotArchiveSource').mockImplementation(async (key, options, owner) => { + const source = await original(key, { ...options, signal: undefined }, owner) + entered.resolve() + await allowOpen.promise + return source + }) + await transport.start(request) + await entered.promise + let closed = false + const pending = server.close().then(() => { + closed = true + }) + expect(server.close()).toBe(Reflect.get(server, 'closing')) + expect(() => server.start()).toThrow('closing') + await new Promise(resolve => setImmediate(resolve)) + expect(closed).toBe(false) + expect(server.server.listening).toBe(false) + allowOpen.resolve() + await pending + expect(closed).toBe(true) + expect(Reflect.get(fixture.storage, 'snapshotSyncSource')).toBeUndefined() + expect(await fixture.storage.knex('snapshot_archive_capacity').first()).toMatchObject({ + archives: 0, + reservedBytes: 0 + }) + // The historical reusable-server lifecycle still works after completed close. + server.start() + if (!server.server.listening) await new Promise(resolve => server.server.once('listening', resolve)) + expect(server.server.listening).toBe(true) + } finally { + allowOpen.resolve() + await fixture.close() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveHttpFixtureCleanup.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveHttpFixtureCleanup.test.ts new file mode 100644 index 000000000..cb934e421 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveHttpFixtureCleanup.test.ts @@ -0,0 +1,93 @@ +import type { WalletInterface } from '@bsv/sdk' +import { snapshotHttpFixture, gate } from '../../../../test/utils/snapshotArchiveHttpFixtures' +import { StorageClient } from '../../remoting/StorageClient' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' + +afterEach(() => jest.restoreAllMocks()) + +test.each(['sync-error', 'reject-error', 'reject-undefined', 'resolve-too-soon'] as const)( + 'fixture owns native resources and drains an opening capture when tested close is %s', + async variant => { + const fixture = await snapshotHttpFixture() + const allowOpen = gate(), + entered = gate(), + failure = new Error('synthetic tested close failure') + let pending: Promise<{ failed: boolean; reason?: unknown }> | undefined + let restored: (() => void) | undefined + let physicallyClosed = false, + sourceClosureFailed = false + try { + const { server, url } = await fixture.serve(), + second = await fixture.serve() + const native = (await fixture.storage.knex.client.acquireConnection()) as { open: boolean } + await fixture.storage.knex.client.releaseConnection(native) + const client = new StorageClient(fixture.wallet as unknown as WalletInterface, url) + const transport = (await client.getSnapshotArchiveTransport(fixture.identityKey))! + const offer = await transport.offer() + const fields = { + version: 1 as const, + nonce: 'e'.repeat(64), + notAfter: offer.serverTime + 300000, + maxBytes: 32768 + } + const original = fixture.storage.openSnapshotArchiveSource.bind(fixture.storage) + jest + .spyOn(fixture.storage, 'openSnapshotArchiveSource') + .mockImplementation(async (key, options, owner) => { + const source = await original(key, { ...options, signal: undefined }, owner) + if (source === undefined) throw new Error('Native archive source was unavailable') + void source.closed.then( + () => { + physicallyClosed = true + }, + () => { + sourceClosureFailed = true + } + ) + entered.resolve() + await allowOpen.promise + return source + }) + await transport.start({ ...fields, requestId: snapshotArchiveRequestId(fields) }) + await entered.promise + const tested = jest.spyOn(server, 'close').mockImplementation(() => { + if (variant === 'sync-error') throw failure + if (variant === 'reject-error') return Promise.reject(failure) + if (variant === 'reject-undefined') return Promise.reject(undefined) + return Promise.resolve() + }) + restored = () => tested.mockRestore() + let settled = false + pending = fixture.close().then( + () => { + settled = true + return { failed: false } + }, + reason => { + settled = true + return { failed: true, reason } + } + ) + await new Promise(resolve => setImmediate(resolve)) + expect(settled).toBe(false) + expect(physicallyClosed).toBe(false) + expect(native.open).toBe(true) + expect(server.server.listening).toBe(false) + expect(second.server.server.listening).toBe(false) + allowOpen.resolve() + const result = await pending + expect(result.failed).toBe(variant !== 'resolve-too-soon') + if (variant === 'reject-undefined') expect(result.reason).toBeUndefined() + else if (variant !== 'resolve-too-soon') expect(result.reason).toBe(failure) + expect(sourceClosureFailed).toBe(false) + expect(physicallyClosed).toBe(true) + expect(native.open).toBe(false) + expect(Reflect.get(fixture.storage, 'snapshotSyncSource')).toBeUndefined() + } finally { + allowOpen.resolve() + restored?.() + await pending + await fixture.close() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveOwner.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveOwner.ts new file mode 100644 index 000000000..d598ef769 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveOwner.ts @@ -0,0 +1,46 @@ +import type { Knex } from 'knex' +import { SnapshotArchiveAdmissionLimitError } from './SnapshotArchiveAdmission' +import { snapshotArchiveLimits } from './SnapshotArchive' +import type { SnapshotArchiveRequestOwner } from './SnapshotArchiveRequest' + +const table = 'snapshot_archive_owners' + +export { SnapshotArchiveCleanupPendingError } from './SnapshotArchiveCleanup' + +/** Call only inside the shared capacity-locked claim transaction. */ +export async function reserveSnapshotArchiveOwner( + k: Knex, + owner: SnapshotArchiveRequestOwner, + guarded = false +): Promise { + const rows: Array<{ slot: number }> = await k(table).select('slot').limit(snapshotArchiveLimits.archives) + const occupied = new Set(rows.map(row => row.slot)) + const slot = Array.from({ length: snapshotArchiveLimits.archives }, (_, index) => index).find( + index => !occupied.has(index) + ) + if (slot === undefined) throw new SnapshotArchiveAdmissionLimitError('Snapshot archive source capacity is occupied') + await k(table).insert({ slot, ...owner, archiveId: null, ...(guarded ? { guardVersion: 1 } : {}) }) +} + +/** Associate the source with its archive in the same transaction as begin(). */ +export async function assignSnapshotArchiveOwner( + k: Knex, + owner: SnapshotArchiveRequestOwner, + archiveId: string +): Promise { + if (await k.schema.hasTable(table)) await k(table).where(owner).update({ archiveId }) +} + +/** Local owner acknowledgement follows awaited physical source and pool cleanup. */ +export async function releaseSnapshotArchiveOwner(k: Knex, owner: SnapshotArchiveRequestOwner): Promise { + await k(table).where(owner).delete() +} + +/** Legacy local archive stores can precede this additive source-owner schema. */ +export async function hasSnapshotArchiveOwner( + k: Knex, + key: { identityKey: string; requestId: string } | { identityKey: string; archiveId: string } +): Promise { + if (!(await k.schema.hasTable(table))) return false + return (await k(table).where(key).first('slot')) !== undefined +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.property.test.ts new file mode 100644 index 000000000..54f9d2c52 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.property.test.ts @@ -0,0 +1,85 @@ +import fc from 'fast-check' +import { createHash } from 'node:crypto' +import { + parseSnapshotArchiveRpcInput, + snapshotArchiveMethods, + validateSnapshotArchiveRequestReceipt, + type SnapshotArchiveMethod +} from './SnapshotArchiveProtocol' +import { SnapshotArchiveTransport } from './SnapshotArchiveTransport' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +test('generated protocol requests preserve exact identity, deadlines and terminal states through client validation', async () => { + await fc.assert( + fc.asyncProperty( + fc.uint8Array({ minLength: 32, maxLength: 32 }), + fc.integer({ min: 1, max: 3600000 }), + fc.integer({ min: 4097, max: 32 * 1024 * 1024 }), + fc.constantFrom('building', 'closed', 'failed', 'expired', 'resource-limited', 'ready'), + fc.constantFrom(...snapshotArchiveMethods), + async (bytes, duration, maxBytes, state, method) => { + const nonce = Buffer.from(bytes).toString('hex') + const identityKey = '02' + nonce + const notAfter = 1790726400000 + duration + const requestId = createHash('sha256') + .update(JSON.stringify(['wallet-snapshot-request/1', 1, nonce, notAfter, maxBytes])) + .digest('hex') + const request = { version: 1 as const, nonce, notAfter, maxBytes, requestId } + const readerRequest = { + version: 2 as const, + nonce, + notAfter, + maxBytes, + requestId: createHash('sha256') + .update(JSON.stringify(['wallet-snapshot-reader-request/1', 2, nonce, notAfter, maxBytes])) + .digest('hex') + } + const archiveId = createHash('sha256') + .update('archive:' + nonce) + .digest('hex') + const digest = createHash('sha256') + .update('directory:' + nonce) + .digest('hex') + const receipt = { + version: 1, + requestId, + expiresAt: notAfter, + state, + ...(state === 'ready' ? { archiveId, digest } : {}) + } + const rpc = jest.fn(async (_method: string, _params: unknown[]) => receipt) + const transport = new SnapshotArchiveTransport(rpc, identityKey, 'source', 'test') + expect(await transport.start(request)).toEqual(receipt) + expect(await transport.status(request)).toEqual(receipt) + expect(rpc.mock.calls[0]).toEqual(['startSnapshotArchive', [{ version: 1, identityKey, request }], undefined]) + const extras: Record = { + getSnapshotArchiveReaderOffer: { options: { lifetimeMs: duration, maxBytes } }, + admitSnapshotArchive: { request: readerRequest }, + cancelSnapshotArchiveRequest: { request: readerRequest }, + getSnapshotArchiveOffer: {}, + startSnapshotArchive: { request }, + getSnapshotArchiveStatus: { requestId }, + cancelSnapshotArchive: { requestId }, + getSnapshotArchiveDirectory: { archiveId }, + readSnapshotArchivePage: { archiveId, sequence: duration % 4096 } + } + const extra = extras[method] + const input = { version: 1, identityKey, ...extra } + expect(parseSnapshotArchiveRpcInput(method, [input])).toEqual({ method, identityKey, ...extra }) + expect(() => parseSnapshotArchiveRpcInput(method, [{ ...input, claimToken: nonce }])).toThrow() + expect(() => validateSnapshotArchiveRequestReceipt({ ...receipt, expiresAt: notAfter + 1 }, request)).toThrow() + expect(() => validateSnapshotArchiveRequestReceipt({ ...receipt, requestId: archiveId }, request)).toThrow() + expect(() => validateSnapshotArchiveRequestReceipt({ ...receipt, writerToken: nonce }, request)).toThrow() + } + ) + ) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.test.ts new file mode 100644 index 000000000..176217e31 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.test.ts @@ -0,0 +1,222 @@ +import { + snapshotArchiveCapabilities, + snapshotArchiveMethods, + snapshotArchiveResponseBytes, + parseSnapshotArchiveRpcInput, + validateSnapshotArchiveCapabilities, + validateSnapshotArchiveOffer, + validateSnapshotArchiveRequestReceipt, + type SnapshotArchiveMethod +} from './SnapshotArchiveProtocol' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import { snapshotArchiveReaderRequestId } from './SnapshotArchiveReaderRequest' +import { verifySnapshotArchiveDirectory } from './SnapshotArchiveDirectory' +import { stringifyJsonRpc } from '../../remoting/BinaryJson' +import { fixture, expected, now, rehash } from '../../../../test/utils/snapshotArchiveDirectoryFixtures' + +const identityKey = '02' + 'ab'.repeat(32) +const fields = { version: 1 as const, nonce: 'a'.repeat(64), notAfter: now + 300000, maxBytes: 32768 } +const request = { ...fields, requestId: snapshotArchiveRequestId(fields) } +const offer = { version: 1, serverTime: now, sourceStorageIdentityKey: 'source', sourceSchema: 'schema', chain: 'test' } +const receipt = { version: 1, requestId: request.requestId, expiresAt: request.notAfter, state: 'building' } +const wire = (value: unknown) => + Buffer.byteLength( + stringifyJsonRpc({ jsonrpc: '2.0', id: Number.MAX_SAFE_INTEGER, result: value }, true).replace(/[<>&]/g, '\\u003c') + ) + +test('detaches and freezes validated capability, offer and all exact receipt states', () => { + expect(validateSnapshotArchiveCapabilities({ ...snapshotArchiveCapabilities })).toBe(snapshotArchiveCapabilities) + expect(snapshotArchiveCapabilities).toEqual({ + version: 1, + encoding: 'wallet-snapshot-rows/1', + maxResponseBytes: 2097152, + maxArchiveBytes: 33554432, + maxPageBytes: 1048576, + maxPages: 4096, + maxLifetimeMs: 3600000 + }) + expect(Object.isFrozen(snapshotArchiveCapabilities)).toBe(true) + const validated = validateSnapshotArchiveOffer(offer, 'source', 'test') + expect(validated).toEqual(offer) + expect(validated).not.toBe(offer) + expect(Object.isFrozen(validated)).toBe(true) + for (const state of ['building', 'closed', 'failed', 'expired', 'resource-limited', 'ready']) { + const input = { + ...receipt, + state, + ...(state === 'ready' ? { archiveId: 'b'.repeat(64), digest: 'c'.repeat(64) } : {}) + } + const accepted = validateSnapshotArchiveRequestReceipt(input, request) + expect(accepted).toEqual(input) + expect(accepted).not.toBe(input) + expect(Object.isFrozen(accepted)).toBe(true) + } +}) + +test.each([null, undefined, 1, [], 'protocol'])('rejects non-record protocol values %p', value => { + expect(() => validateSnapshotArchiveCapabilities(value)).toThrow('Invalid snapshot archive protocol') + expect(() => validateSnapshotArchiveOffer(value, 'source', 'test')).toThrow() + expect(() => validateSnapshotArchiveRequestReceipt(value, request)).toThrow() +}) + +test('capability fields are exact, own enumerable data and immutable version-one limits', () => { + for (const field of Object.keys(snapshotArchiveCapabilities)) { + expect(() => validateSnapshotArchiveCapabilities({ ...snapshotArchiveCapabilities, [field]: 'wrong' })).toThrow() + const missing = { ...snapshotArchiveCapabilities } as Record + delete missing[field] + expect(() => validateSnapshotArchiveCapabilities(missing)).toThrow() + Object.defineProperty(missing, field, { value: Reflect.get(snapshotArchiveCapabilities, field) }) + expect(() => validateSnapshotArchiveCapabilities(missing)).toThrow() + } + const accessor = { ...snapshotArchiveCapabilities } + Object.defineProperty(accessor, 'version', { + get: () => { + throw new Error('Getter must not run') + } + }) + expect(() => validateSnapshotArchiveCapabilities(accessor)).toThrow('Invalid snapshot archive protocol') + expect(() => + validateSnapshotArchiveCapabilities({ ...snapshotArchiveCapabilities, [Symbol('extra')]: true }) + ).toThrow() + const inherited = Object.create(snapshotArchiveCapabilities) + Object.defineProperty(inherited, 'placeholder', { value: true }) + expect(() => validateSnapshotArchiveCapabilities(inherited)).toThrow() + expect(() => validateSnapshotArchiveCapabilities({ ...snapshotArchiveCapabilities, extra: true })).toThrow() +}) + +test.each([ + { version: 2 }, + { sourceStorageIdentityKey: 'changed' }, + { chain: 'main' }, + { sourceSchema: '' }, + { sourceSchema: 'x'.repeat(257) }, + { sourceSchema: null }, + { serverTime: -1 }, + { serverTime: 0.1 }, + { serverTime: NaN }, + { serverTime: Infinity }, + { serverTime: Number.MAX_SAFE_INTEGER + 1 }, + { serverTime: '123' } +])('refuses changed or malformed offer %p', changes => { + expect(() => validateSnapshotArchiveOffer({ ...offer, ...changes }, 'source', 'test')).toThrow() +}) + +test('offer integer and schema boundaries are inclusive', () => { + expect( + validateSnapshotArchiveOffer({ ...offer, serverTime: 0, sourceSchema: 'x' }, 'source', 'test').serverTime + ).toBe(0) + expect( + validateSnapshotArchiveOffer( + { ...offer, serverTime: Number.MAX_SAFE_INTEGER, sourceSchema: 'x'.repeat(256) }, + 'source', + 'test' + ).sourceSchema + ).toHaveLength(256) +}) + +test.each([ + { version: 0 }, + { requestId: 'b'.repeat(64) }, + { expiresAt: request.notAfter + 1 }, + { state: 'other' }, + { state: null }, + { claimToken: 'internal' }, + { archiveId: 'a'.repeat(64) } +])('receipt is bound to exact request and fields %p', changes => { + expect(() => validateSnapshotArchiveRequestReceipt({ ...receipt, ...changes }, request)).toThrow() +}) + +test.each(['archiveId', 'digest'])('ready receipt requires a canonical %s', field => { + for (const value of [undefined, '', 'A'.repeat(64), 'g'.repeat(64), 'a'.repeat(63), 'a'.repeat(65), 1]) { + expect(() => + validateSnapshotArchiveRequestReceipt( + { ...receipt, state: 'ready', archiveId: 'b'.repeat(64), digest: 'c'.repeat(64), [field]: value }, + request + ) + ).toThrow() + } +}) + +test('every RPC has exact typed arguments and no ownership-token parameter', () => { + const readerFields = { ...fields, version: 2 as const } + const readerRequest = { ...readerFields, requestId: snapshotArchiveReaderRequestId(readerFields) } + const extras: Record> = { + getSnapshotArchiveOffer: {}, + startSnapshotArchive: { request }, + getSnapshotArchiveStatus: { requestId: request.requestId }, + cancelSnapshotArchive: { requestId: request.requestId }, + getSnapshotArchiveDirectory: { archiveId: 'b'.repeat(64) }, + readSnapshotArchivePage: { archiveId: 'b'.repeat(64), sequence: 0 }, + getSnapshotArchiveReaderOffer: { options: { lifetimeMs: 300000, maxBytes: 32768 } }, + admitSnapshotArchive: { request: readerRequest }, + cancelSnapshotArchiveRequest: { request: readerRequest } + } + for (const method of snapshotArchiveMethods) { + const extra = extras[method] + const input = { version: 1, identityKey, ...extra } + expect(parseSnapshotArchiveRpcInput(method, [input])).toEqual({ method, identityKey, ...extra }) + for (const params of [ + [], + [input, input], + [null], + [{ ...input, writerToken: 'x' }], + [{ ...input, version: 2 }], + [{ ...input, identityKey: '' }], + [{ ...input, identityKey: 1 }] + ]) { + expect(() => parseSnapshotArchiveRpcInput(method, params)).toThrow() + } + } +}) + +test('strong reader methods and legacy admission reject each other’s request version', () => { + const readerFields = { ...fields, version: 2 as const } + const readerRequest = { ...readerFields, requestId: snapshotArchiveReaderRequestId(readerFields) } + expect(() => + parseSnapshotArchiveRpcInput('startSnapshotArchive', [{ version: 1, identityKey, request: readerRequest }]) + ).toThrow() + for (const method of ['admitSnapshotArchive', 'cancelSnapshotArchiveRequest'] as const) { + expect(() => parseSnapshotArchiveRpcInput(method, [{ version: 1, identityKey, request }])).toThrow() + } +}) + +test.each([-1, 0.1, 4096, Infinity, NaN, '0', null])('rejects page sequence %p before dispatch', sequence => { + expect(() => + parseSnapshotArchiveRpcInput('readSnapshotArchivePage', [ + { version: 1, identityKey, archiveId: 'b'.repeat(64), sequence } + ]) + ).toThrow() +}) + +test('a maximum valid directory and maximum binary page fit the escaped RPC response ceiling', () => { + const { directory, binding } = fixture(4083) + binding.sourceStorage.storageName = '<>&'.repeat(85) + binding.sourceSchema = '<>&'.repeat(85) + directory.bindingJson = JSON.stringify(binding) + rehash(directory) + expect(verifySnapshotArchiveDirectory(directory, expected, now).receipts).toHaveLength(4096) + expect(wire(directory)).toBeLessThan(snapshotArchiveResponseBytes) + expect(wire({ ...directory.receipts[0], bytes: new Uint8Array(1024 * 1024) })).toBeLessThan( + snapshotArchiveResponseBytes + ) + expect( + parseSnapshotArchiveRpcInput('readSnapshotArchivePage', [ + { version: 1, identityKey, archiveId: 'b'.repeat(64), sequence: 4095 } + ]) + ).toMatchObject({ sequence: 4095 }) +}) + +test('identity matching anchors the complete string, and schema values cannot coerce from objects', () => { + for (const value of ['prefix' + identityKey, identityKey + 'suffix', Object(identityKey)]) { + expect(() => parseSnapshotArchiveRpcInput('getSnapshotArchiveOffer', [{ version: 1, identityKey: value }])).toThrow( + 'Invalid snapshot archive protocol' + ) + } + expect(() => validateSnapshotArchiveOffer({ ...offer, sourceSchema: { length: 12 } }, 'source', 'test')).toThrow( + 'Invalid snapshot archive protocol' + ) + const missing = { ...snapshotArchiveCapabilities } as Record + delete missing.version + missing.placeholder = 1 + expect(() => validateSnapshotArchiveCapabilities(missing)).toThrow('Invalid snapshot archive protocol') +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.ts new file mode 100644 index 000000000..51e7c3519 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveProtocol.ts @@ -0,0 +1,203 @@ +import { + validateSnapshotArchiveReaderOptions, + type SnapshotArchiveReaderOptions, + parseSnapshotArchiveReaderRequest, + type SnapshotArchiveReaderRequest +} from './SnapshotArchiveReaderRequest' +import { snapshotArchiveLimits } from './SnapshotArchive' +import { snapshotArchiveEncoding } from './SnapshotArchiveDirectory' +import { + parseSnapshotArchiveRequest, + type SnapshotArchiveRequest, + type SnapshotArchiveRequestReceipt +} from './SnapshotArchiveRequest' + +/** Includes JSON-RPC framing, compact binary tags and HTML escaping. */ +export const snapshotArchiveResponseBytes = 2 * 1024 * 1024 +export const snapshotArchiveRequestBytes = 4096 +export const snapshotArchiveCapabilities = Object.freeze({ + version: 1 as const, + encoding: snapshotArchiveEncoding, + maxResponseBytes: snapshotArchiveResponseBytes, + maxArchiveBytes: snapshotArchiveLimits.archiveBytes, + maxPageBytes: snapshotArchiveLimits.pageBytes, + maxPages: snapshotArchiveLimits.pages, + maxLifetimeMs: snapshotArchiveLimits.lifetimeMs +}) +export type SnapshotArchiveCapabilities = typeof snapshotArchiveCapabilities + +export const snapshotArchiveMethods = Object.freeze([ + 'getSnapshotArchiveOffer', + 'startSnapshotArchive', + 'getSnapshotArchiveStatus', + 'getSnapshotArchiveDirectory', + 'readSnapshotArchivePage', + 'cancelSnapshotArchive', + 'getSnapshotArchiveReaderOffer', + 'admitSnapshotArchive', + 'cancelSnapshotArchiveRequest' +] as const) +export type SnapshotArchiveMethod = (typeof snapshotArchiveMethods)[number] + +export interface SnapshotArchiveOffer { + version: 1 + serverTime: number + sourceStorageIdentityKey: string + sourceSchema: string + chain: 'main' | 'test' +} + +function invalid(): never { + throw new TypeError('Invalid snapshot archive protocol value') +} + +function record(input: unknown, names: readonly string[]): Record { + if (input === null || typeof input !== 'object' || Array.isArray(input)) invalid() + if (Reflect.ownKeys(input).length !== names.length) invalid() + const result: Record = {} + for (const name of names) { + const property = Object.getOwnPropertyDescriptor(input, name) + if (property === undefined || !('value' in property) || !property.enumerable) invalid() + result[name] = property.value + } + return result +} + +function integer(input: unknown, minimum: number, maximum = Number.MAX_SAFE_INTEGER): number { + if (typeof input !== 'number' || !Number.isSafeInteger(input) || input < minimum || input > maximum) invalid() + return input +} + +function digest(input: unknown): string { + if (typeof input !== 'string' || !/^[0-9a-f]{64}$/.test(input)) invalid() + return input +} + +export function validateSnapshotArchiveCapabilities(input: unknown): SnapshotArchiveCapabilities { + const value = record(input, Object.keys(snapshotArchiveCapabilities)) + for (const [key, expected] of Object.entries(snapshotArchiveCapabilities)) { + if (value[key] !== expected) invalid() + } + return snapshotArchiveCapabilities +} + +export type SnapshotArchiveRpcInput = + | { method: 'getSnapshotArchiveReaderOffer'; identityKey: string; options: Readonly } + | { + method: 'admitSnapshotArchive' | 'cancelSnapshotArchiveRequest' + identityKey: string + request: Readonly + } + | { method: 'getSnapshotArchiveOffer'; identityKey: string } + | { method: 'startSnapshotArchive'; identityKey: string; request: Readonly } + | { method: 'getSnapshotArchiveStatus' | 'cancelSnapshotArchive'; identityKey: string; requestId: string } + | { method: 'getSnapshotArchiveDirectory'; identityKey: string; archiveId: string } + | { method: 'readSnapshotArchivePage'; identityKey: string; archiveId: string; sequence: number } + +function requestFields(method: SnapshotArchiveMethod): string[] { + switch (method) { + case 'getSnapshotArchiveReaderOffer': + return ['options'] + case 'admitSnapshotArchive': + case 'cancelSnapshotArchiveRequest': + case 'startSnapshotArchive': + return ['request'] + case 'getSnapshotArchiveStatus': + case 'cancelSnapshotArchive': + return ['requestId'] + case 'getSnapshotArchiveDirectory': + return ['archiveId'] + case 'readSnapshotArchivePage': + return ['archiveId', 'sequence'] + default: + return [] + } +} + +/** Exact method-specific data only; ownership tokens have no wire representation. */ +export function parseSnapshotArchiveRpcInput( + method: SnapshotArchiveMethod, + params: unknown[] +): SnapshotArchiveRpcInput { + if (params.length !== 1) invalid() + const input = record(params[0], ['version', 'identityKey', ...requestFields(method)]) + if (input.version !== 1 || typeof input.identityKey !== 'string' || !/^(02|03)[0-9a-f]{64}$/.test(input.identityKey)) + invalid() + const identityKey = input.identityKey + switch (method) { + case 'getSnapshotArchiveReaderOffer': + return { method, identityKey, options: validateSnapshotArchiveReaderOptions(input.options) } + case 'admitSnapshotArchive': + case 'cancelSnapshotArchiveRequest': + return { method, identityKey, request: parseSnapshotArchiveReaderRequest(input.request) } + case 'getSnapshotArchiveOffer': + return { method, identityKey } + case 'startSnapshotArchive': + return { method, identityKey, request: parseSnapshotArchiveRequest(input.request) } + case 'getSnapshotArchiveStatus': + case 'cancelSnapshotArchive': + return { method, identityKey, requestId: digest(input.requestId) } + case 'getSnapshotArchiveDirectory': + return { method, identityKey, archiveId: digest(input.archiveId) } + case 'readSnapshotArchivePage': + return { + method, + identityKey, + archiveId: digest(input.archiveId), + sequence: integer(input.sequence, 0, snapshotArchiveLimits.pages - 1) + } + } +} + +export function validateSnapshotArchiveOffer( + input: unknown, + storageIdentityKey: string, + chain: 'main' | 'test' +): Readonly { + const value = record(input, ['version', 'serverTime', 'sourceStorageIdentityKey', 'sourceSchema', 'chain']) + if ( + value.version !== 1 || + value.sourceStorageIdentityKey !== storageIdentityKey || + value.chain !== chain || + typeof value.sourceSchema !== 'string' || + value.sourceSchema.length < 1 || + value.sourceSchema.length > 256 + ) + invalid() + return Object.freeze({ + version: 1, + serverTime: integer(value.serverTime, 0), + sourceStorageIdentityKey: storageIdentityKey, + sourceSchema: value.sourceSchema, + chain + }) +} + +export function validateSnapshotArchiveRequestReceipt( + input: unknown, + request: Pick +): Readonly { + const state = + input !== null && typeof input === 'object' ? Object.getOwnPropertyDescriptor(input, 'state')?.value : undefined + const value = record(input, [ + 'version', + 'requestId', + 'expiresAt', + 'state', + ...(state === 'ready' ? ['archiveId', 'digest'] : []) + ]) + if ( + value.version !== 1 || + value.requestId !== request.requestId || + value.expiresAt !== request.notAfter || + !['building', 'ready', 'closed', 'failed', 'expired', 'resource-limited'].includes(state) + ) + invalid() + return Object.freeze({ + version: 1, + requestId: request.requestId, + expiresAt: request.notAfter, + state, + ...(state === 'ready' ? { archiveId: digest(value.archiveId), digest: digest(value.digest) } : {}) + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderClient.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderClient.test.ts new file mode 100644 index 000000000..27b48246b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderClient.test.ts @@ -0,0 +1,113 @@ +import type { WalletInterface } from '@bsv/sdk' +import { StorageClientBase, type StorageClientOptions } from '../../remoting/StorageClientBase' +import type { WalletReadSnapshot } from '../WalletReadSnapshot' +import { snapshotArchiveCapabilities } from './SnapshotArchiveProtocol' +import * as RemoteReader from './openRemoteSnapshot' +import { SnapshotCancelledError } from '../SnapshotCancelledError' + +const identityKey = '02' + '11'.repeat(32) +class ReaderClient extends StorageClientBase { + readonly request = jest.fn, [string, unknown[]]>() + constructor(options: StorageClientOptions = {}) { + super({} as WalletInterface, 'https://storage.example.test', options) + } + protected async rpcCall(method: string, params: unknown[]): Promise { + return (await this.request(method, params)) as T + } +} +const settings = () => ({ + storageIdentityKey: 'reader-source', + chain: 'test', + snapshotArchive: { ...snapshotArchiveCapabilities }, + snapshotArchiveReaderVersion: 1 +}) +afterEach(() => jest.restoreAllMocks()) + +test('a source adapter exists before availability and negotiates the first open with detached capability state', async () => { + const client = new ReaderClient() + client.request.mockResolvedValue(settings()) + const source = client.getSnapshotSync()! + expect(client.request).not.toHaveBeenCalled() + expect(source.fallbackOnResourceError).toBe(false) + expect(await source.supportsDestination()).toBe(false) + const view = {} as WalletReadSnapshot + const opening = jest.spyOn(RemoteReader, 'openRemoteSnapshot').mockResolvedValue(view) + const options = { lifetimeMs: 1234, signal: new AbortController().signal } + expect(await source.openSource(identityKey, options)).toBe(view) + expect(client.request.mock.calls).toEqual([['makeAvailable', []]]) + expect(opening).toHaveBeenCalledWith(expect.objectContaining({ supportsReader: true }), options) + client.settings!.snapshotArchiveReaderVersion = undefined + expect((await client.getSnapshotArchiveTransport(identityKey))!.supportsReader).toBe(true) + for (const operation of [ + () => source.begin(undefined as never, undefined), + () => source.checkpoint('', ''), + () => source.prepare(undefined as never, undefined as never) + ]) + await expect(operation()).rejects.toThrow('Remote snapshot destination is unavailable') + expect(client.request).toHaveBeenCalledTimes(1) +}) + +test.each(['legacy', 'archive-only'] as const)( + '%s peers decline before any reader request and cannot be upgraded by caller mutation', + async variant => { + const client = new ReaderClient() + const advertised = settings() + Reflect.deleteProperty(advertised, 'snapshotArchiveReaderVersion') + if (variant === 'legacy') Reflect.deleteProperty(advertised, 'snapshotArchive') + client.request.mockResolvedValue(advertised) + expect(await client.getSnapshotSync()!.openSource(identityKey)).toBeUndefined() + client.settings!.snapshotArchiveReaderVersion = 1 + const transport = await client.getSnapshotArchiveTransport(identityKey) + expect(transport?.supportsReader).not.toBe(true) + expect(client.request.mock.calls).toEqual([['makeAvailable', []]]) + } +) + +test('the existing client rollback option disables the source adapter before network I/O', () => { + const client = new ReaderClient({ snapshotArchives: false }) + expect(client.getSnapshotSync()).toBeUndefined() + expect(client.request).not.toHaveBeenCalled() +}) + +test.each([undefined, null, 0, 2, '1', true])( + 'an invalid reader advertisement %p rejects availability', + async version => { + const client = new ReaderClient() + client.request.mockResolvedValue({ ...settings(), snapshotArchiveReaderVersion: version }) + await expect(client.getSnapshotSync()!.openSource(identityKey)).rejects.toThrow('invalid settings') + expect(client.isAvailable()).toBe(false) + } +) + +test('reader advertisement requires the original archive capability', async () => { + const client = new ReaderClient() + const advertised = settings() + Reflect.deleteProperty(advertised, 'snapshotArchive') + client.request.mockResolvedValue(advertised) + await expect(client.makeAvailable()).rejects.toThrow('invalid settings') +}) + +test.each(['cancelled', 'live-signal', 'ordinary', 'inherited-code', 'accessor-code'] as const)( + 'authenticated transport %s classification preserves errors outside explicit local cancellation', + async variant => { + const client = new ReaderClient() + client.request.mockResolvedValue(settings()) + const transport = (await client.getSnapshotArchiveTransport(identityKey))! + const controller = new AbortController() + const failure = new Error('synthetic authenticated transport refusal') + const getter = jest.fn(() => 'ERR_PAYMENT_CANCELLED') + if (variant === 'cancelled' || variant === 'live-signal') + Object.defineProperty(failure, 'code', { value: 'ERR_PAYMENT_CANCELLED' }) + if (variant === 'inherited-code') + Object.setPrototypeOf(failure, Object.create(Error.prototype, { code: { value: 'ERR_PAYMENT_CANCELLED' } })) + if (variant === 'accessor-code') Object.defineProperty(failure, 'code', { get: getter }) + if (variant !== 'live-signal') controller.abort() + const fetch = jest.fn().mockRejectedValue(failure) + Reflect.set(client, 'snapshotAuthClient', { fetch }) + const opening = transport.readerOffer({ lifetimeMs: 300000, maxBytes: 32768 }, controller.signal) + if (variant === 'cancelled') await expect(opening).rejects.toBeInstanceOf(SnapshotCancelledError) + else await expect(opening).rejects.toBe(failure) + expect(fetch).toHaveBeenCalledTimes(1) + expect(getter).not.toHaveBeenCalled() + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderOffer.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderOffer.test.ts new file mode 100644 index 000000000..508b18635 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderOffer.test.ts @@ -0,0 +1,129 @@ +import { snapshotArchiveLimits } from './SnapshotArchive' +import { snapshotArchiveReaderRequestId } from './SnapshotArchiveReaderRequest' +import { validateSnapshotArchiveReaderOffer, validateSnapshotArchiveReaderOptions } from './SnapshotArchiveReaderOffer' + +const options = { lifetimeMs: 300000, maxBytes: 32768 } +const storage = 'original-storage' +function fixture() { + const fields = { version: 2 as const, nonce: 'a'.repeat(64), notAfter: 1300000, maxBytes: 32768 } + return { + version: 1, + outcome: 'offered', + offer: { + version: 1, + serverTime: 1000000, + sourceStorageIdentityKey: storage, + sourceSchema: 'original-schema', + chain: 'test' + }, + request: { ...fields, requestId: snapshotArchiveReaderRequestId(fields) } + } +} +const parse = (value: unknown) => validateSnapshotArchiveReaderOffer(value, options, storage, 'test') + +test('reader offer retains exactly the authenticated options, source and immutable request', () => { + const input = fixture() + const output = parse(input) + expect(output).toEqual(input) + expect(output).not.toBe(input) + if (output.outcome !== 'offered') throw new Error('Expected offered fixture') + expect(output.request).not.toBe(input.request) + expect(output.offer).not.toBe(input.offer) + input.offer.sourceSchema = 'later caller mutation' + input.request.nonce = 'b'.repeat(64) + expect(output.offer.sourceSchema).toBe('original-schema') + expect(output.request.nonce).toBe('a'.repeat(64)) + expect(Object.isFrozen(output)).toBe(true) + expect(Object.isFrozen(output.offer)).toBe(true) + expect(Object.isFrozen(output.request)).toBe(true) + expect(parse({ version: 1, outcome: 'resource-limited' })).toEqual({ version: 1, outcome: 'resource-limited' }) +}) + +test('a refusal carries no request that could have been admitted', () => { + for (const name of ['request', 'offer', 'requestId', 'expiresAt', 'claimToken']) { + expect(() => parse({ version: 1, outcome: 'resource-limited', [name]: fixture().request })).toThrow( + 'Invalid snapshot archive reader offer' + ) + } + for (const outcome of ['accepted', 'building', 'ready', undefined, null, 1]) { + expect(() => parse({ version: 1, outcome })).toThrow('Invalid snapshot archive reader offer') + } +}) + +test('offer framing rejects altered versions, hidden fields and accessors without evaluating them', () => { + for (const input of [null, [], 'offered', 1, {}, { ...fixture(), version: 2 }, { ...fixture(), extra: true }]) { + expect(() => parse(input)).toThrow('Invalid snapshot archive reader offer') + } + for (const name of ['version', 'outcome', 'offer', 'request']) { + const getter = jest.fn(() => { + throw new Error('must not evaluate property') + }) + const input = fixture() + Object.defineProperty(input, name, { get: getter, enumerable: true }) + expect(() => parse(input)).toThrow('Invalid snapshot archive reader offer') + expect(getter).not.toHaveBeenCalled() + const hidden = fixture() + Object.defineProperty(hidden, name, { value: hidden[name as keyof typeof hidden], enumerable: false }) + expect(() => parse(hidden)).toThrow('Invalid snapshot archive reader offer') + const missing = { ...fixture() } as Record + delete missing[name] + expect(() => parse(missing)).toThrow('Invalid snapshot archive reader offer') + } + const symbol = fixture() + Object.defineProperty(symbol, Symbol('extra'), { value: true }) + expect(() => parse(symbol)).toThrow('Invalid snapshot archive reader offer') +}) + +test('valid request hashes cannot change negotiated time, budget, source schema shape or chain', () => { + for (const change of [{ notAfter: 1300001 }, { notAfter: 1299999 }, { maxBytes: 32769 }, { maxBytes: 32767 }]) { + const input = fixture() + Object.assign(input.request, change) + input.request.requestId = snapshotArchiveReaderRequestId(input.request) + expect(() => parse(input)).toThrow() + } + for (const change of [ + { serverTime: 999999 }, + { sourceStorageIdentityKey: 'substituted' }, + { sourceSchema: '' }, + { chain: 'main' } + ]) { + const input = fixture() + Object.assign(input.offer, change) + expect(() => parse(input)).toThrow() + } + const changedHash = fixture() + changedHash.request.requestId = 'b'.repeat(64) + expect(() => parse(changedHash)).toThrow() +}) + +test('reader options admit exact integer bounds and reject unbounded or getter-backed input', () => { + const minimum = { lifetimeMs: 1, maxBytes: snapshotArchiveLimits.headerCharge + 1 } + const maximum = { lifetimeMs: snapshotArchiveLimits.lifetimeMs, maxBytes: snapshotArchiveLimits.archiveBytes } + expect(validateSnapshotArchiveReaderOptions(minimum)).toEqual(minimum) + expect(validateSnapshotArchiveReaderOptions(maximum)).toEqual(maximum) + for (const lifetimeMs of [0, -1, 1.5, Number.NaN, Infinity, snapshotArchiveLimits.lifetimeMs + 1, '1']) { + expect(() => validateSnapshotArchiveReaderOptions({ ...options, lifetimeMs })).toThrow( + 'Invalid snapshot archive reader request' + ) + } + for (const maxBytes of [ + snapshotArchiveLimits.headerCharge, + snapshotArchiveLimits.archiveBytes + 1, + 1.5, + Number.NaN, + Infinity, + '32768' + ]) { + expect(() => validateSnapshotArchiveReaderOptions({ ...options, maxBytes })).toThrow( + 'Invalid snapshot archive reader request' + ) + } + for (const input of [null, [], {}, { lifetimeMs: 1 }, { maxBytes: 32768 }, { ...options, extra: true }]) { + expect(() => validateSnapshotArchiveReaderOptions(input)).toThrow('Invalid snapshot archive reader request') + } + const getter = jest.fn(() => 1) + expect(() => + validateSnapshotArchiveReaderOptions(Object.defineProperty({ ...options }, 'lifetimeMs', { get: getter })) + ).toThrow('Invalid snapshot archive reader request') + expect(getter).not.toHaveBeenCalled() +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderOffer.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderOffer.ts new file mode 100644 index 000000000..94b8c827e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderOffer.ts @@ -0,0 +1,56 @@ +import { validateSnapshotArchiveOffer, type SnapshotArchiveOffer } from './SnapshotArchiveProtocol' +import { + validateSnapshotArchiveReaderRequest, + validateSnapshotArchiveReaderOptions, + type SnapshotArchiveReaderRequest, + type SnapshotArchiveReaderOptions +} from './SnapshotArchiveReaderRequest' +export { validateSnapshotArchiveReaderOptions, type SnapshotArchiveReaderOptions } from './SnapshotArchiveReaderRequest' + +/** A refused offer has no request and cannot have started a capture. */ +export type SnapshotArchiveReaderOffer = + | { + version: 1 + outcome: 'offered' + offer: Readonly + request: Readonly + } + | { version: 1; outcome: 'resource-limited' } + +function invalid(): never { + throw new TypeError('Invalid snapshot archive reader offer') +} + +function record(input: unknown, names: readonly string[]): Record { + if (input === null || typeof input !== 'object' || Array.isArray(input)) invalid() + if (Reflect.ownKeys(input).length !== names.length) invalid() + const result: Record = {} + for (const name of names) { + const field = Object.getOwnPropertyDescriptor(input, name) + if (field === undefined || !('value' in field) || !field.enumerable) invalid() + result[name] = field.value + } + return result +} + +export function validateSnapshotArchiveReaderOffer( + input: unknown, + options: SnapshotArchiveReaderOptions, + storageIdentityKey: string, + chain: 'main' | 'test' +): Readonly { + const expected = validateSnapshotArchiveReaderOptions(options) + const outcome = + input !== null && typeof input === 'object' ? Object.getOwnPropertyDescriptor(input, 'outcome')?.value : undefined + const fields = record( + input, + outcome === 'offered' ? ['version', 'outcome', 'offer', 'request'] : ['version', 'outcome'] + ) + if (fields.version !== 1) invalid() + if (fields.outcome === 'resource-limited') return Object.freeze({ version: 1, outcome: 'resource-limited' }) + if (fields.outcome !== 'offered') invalid() + const offer = validateSnapshotArchiveOffer(fields.offer, storageIdentityKey, chain) + const request = validateSnapshotArchiveReaderRequest(fields.request, offer.serverTime) + if (request.notAfter !== offer.serverTime + expected.lifetimeMs || request.maxBytes !== expected.maxBytes) invalid() + return Object.freeze({ version: 1, outcome: 'offered', offer, request }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderRequest.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderRequest.test.ts new file mode 100644 index 000000000..97ab6120e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderRequest.test.ts @@ -0,0 +1,134 @@ +import { createHash } from 'node:crypto' +import { parseSnapshotArchiveRequest, snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import { + parseSnapshotArchiveReaderRequest, + snapshotArchiveReaderRequestId, + validateSnapshotArchiveReaderRequest +} from './SnapshotArchiveReaderRequest' +import { snapshotArchiveLimits } from './SnapshotArchive' + +const fields = { version: 2 as const, nonce: 'a'.repeat(64), notAfter: 1300000, maxBytes: 32768 } +function fixture() { + return { ...fields, requestId: snapshotArchiveReaderRequestId(fields) } +} + +test('independent SHA-256 fixes the reader domain and both parser paths reject cross-protocol requests', () => { + const expected = createHash('sha256') + .update(JSON.stringify(['wallet-snapshot-reader-request/1', 2, 'a'.repeat(64), 1300000, 32768])) + .digest('hex') + expect(snapshotArchiveReaderRequestId(fields)).toBe(expected) + const reader = fixture() + expect(parseSnapshotArchiveReaderRequest(reader)).toEqual(reader) + expect(() => parseSnapshotArchiveRequest(reader)).toThrow() + expect(() => parseSnapshotArchiveRequest({ ...reader, version: 1 })).toThrow() + const legacyFields = { ...fields, version: 1 as const } + const legacy = { ...legacyFields, requestId: snapshotArchiveRequestId(legacyFields) } + expect(legacy.requestId).not.toBe(reader.requestId) + expect(parseSnapshotArchiveRequest(legacy)).toEqual(legacy) + expect(() => parseSnapshotArchiveReaderRequest(legacy)).toThrow() + expect(() => parseSnapshotArchiveReaderRequest({ ...legacy, version: 2 })).toThrow() +}) + +test('reader request parsing detaches every field before asynchronous admission', () => { + const input = fixture() + const parsed = parseSnapshotArchiveReaderRequest(input) + input.nonce = 'b'.repeat(64) + input.notAfter++ + input.maxBytes++ + input.requestId = 'b'.repeat(64) + expect(parsed).toEqual(fixture()) + expect(Object.isFrozen(parsed)).toBe(true) +}) + +test('reader request exact fields reject altered identities, invalid bounds and hidden/getter values', () => { + const changes = [ + { version: 1 }, + { nonce: 'a'.repeat(63) }, + { nonce: 'A'.repeat(64) }, + { nonce: 1 }, + { notAfter: 0 }, + { notAfter: 1.5 }, + { notAfter: Number.MAX_SAFE_INTEGER + 1 }, + { notAfter: '1300000' }, + { maxBytes: snapshotArchiveLimits.headerCharge }, + { maxBytes: snapshotArchiveLimits.archiveBytes + 1 }, + { maxBytes: Number.NaN }, + { maxBytes: '32768' }, + { requestId: 'b'.repeat(64) }, + { extra: true } + ] + for (const change of changes) + expect(() => parseSnapshotArchiveReaderRequest({ ...fixture(), ...change })).toThrow( + 'Invalid snapshot archive reader request' + ) + for (const input of [undefined, null, 1, [], {}, 'request']) + expect(() => parseSnapshotArchiveReaderRequest(input)).toThrow('Invalid snapshot archive reader request') + for (const key of Object.keys(fixture())) { + const missing = { ...fixture() } as Record + delete missing[key] + expect(() => parseSnapshotArchiveReaderRequest(missing)).toThrow('Invalid snapshot archive reader request') + const getter = jest.fn(() => 1) + expect(() => + parseSnapshotArchiveReaderRequest(Object.defineProperty(fixture(), key, { get: getter, enumerable: true })) + ).toThrow('Invalid snapshot archive reader request') + expect(getter).not.toHaveBeenCalled() + const hidden = fixture() + Object.defineProperty(hidden, key, { enumerable: false }) + expect(() => parseSnapshotArchiveReaderRequest(hidden)).toThrow('Invalid snapshot archive reader request') + } +}) + +test('database-clock deadline validation never extends a reader after expiry', () => { + expect(validateSnapshotArchiveReaderRequest(fixture(), 1000000)).toEqual(fixture()) + expect(validateSnapshotArchiveReaderRequest(fixture(), fields.notAfter - 1)).toEqual(fixture()) + for (const now of [fields.notAfter, fields.notAfter + 1, -1, Number.NaN, Infinity, 1.5]) { + expect(() => validateSnapshotArchiveReaderRequest(fixture(), now)).toThrow( + 'Invalid snapshot archive reader request' + ) + } + const long = { ...fields, notAfter: 1000000 + snapshotArchiveLimits.lifetimeMs } + expect( + validateSnapshotArchiveReaderRequest({ ...long, requestId: snapshotArchiveReaderRequestId(long) }, 1000000).notAfter + ).toBe(long.notAfter) + long.notAfter++ + expect(() => + validateSnapshotArchiveReaderRequest({ ...long, requestId: snapshotArchiveReaderRequestId(long) }, 1000000) + ).toThrow('Invalid snapshot archive reader request') +}) + +test('valid independent digests do not authorize malformed request fields or exclude inclusive bounds', () => { + const signed = (changes: Record) => { + const value = { ...fields, ...changes } + const requestId = createHash('sha256') + .update( + JSON.stringify(['wallet-snapshot-reader-request/1', value.version, value.nonce, value.notAfter, value.maxBytes]) + ) + .digest('hex') + return { ...value, requestId } + } + for (const change of [ + { nonce: 'x' + fields.nonce }, + { nonce: fields.nonce + 'x' }, + { nonce: fields.nonce + '\n' }, + { notAfter: 0 }, + { notAfter: -1 }, + { notAfter: 1.5 }, + { notAfter: '1300000' }, + { maxBytes: snapshotArchiveLimits.headerCharge }, + { maxBytes: snapshotArchiveLimits.headerCharge - 1 }, + { maxBytes: snapshotArchiveLimits.archiveBytes + 1 }, + { maxBytes: '32768' } + ]) { + expect(() => parseSnapshotArchiveReaderRequest(signed(change))).toThrow('Invalid snapshot archive reader request') + } + for (const maxBytes of [snapshotArchiveLimits.headerCharge + 1, snapshotArchiveLimits.archiveBytes]) { + const value = signed({ notAfter: 1, maxBytes }) + expect(parseSnapshotArchiveReaderRequest(value)).toEqual(value) + expect(validateSnapshotArchiveReaderRequest(value, 0)).toEqual(value) + } + for (const name of Object.keys(fixture())) { + const missing: Record = { ...fixture(), replacement: 1 } + delete missing[name] + expect(() => parseSnapshotArchiveReaderRequest(missing)).toThrow('Invalid snapshot archive reader request') + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderRequest.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderRequest.ts new file mode 100644 index 000000000..d20d3d136 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveReaderRequest.ts @@ -0,0 +1,119 @@ +import { syncTransferDigest } from '../../remoting/SyncTransfer' +import { snapshotArchiveLimits } from './SnapshotArchive' + +/** Server-issued reader admission only. An absent request can never create a capture. */ +export interface SnapshotArchiveReaderRequest { + version: 2 + nonce: string + notAfter: number + maxBytes: number + requestId: string +} + +function invalid(): never { + throw new TypeError('Invalid snapshot archive reader request') +} + +export interface SnapshotArchiveReaderOptions { + lifetimeMs: number + maxBytes: number +} + +function record(input: unknown, names: readonly string[]): Record { + if ( + input === null || + typeof input !== 'object' || + Array.isArray(input) || + Reflect.ownKeys(input).length !== names.length + ) + invalid() + const fields: Record = {} + for (const name of names) { + const property = Object.getOwnPropertyDescriptor(input, name) + if (property === undefined || !('value' in property) || !property.enumerable) invalid() + fields[name] = property.value + } + return fields +} + +export function validateSnapshotArchiveReaderOptions(input: unknown): Readonly { + const fields = record(input, ['lifetimeMs', 'maxBytes']) + const { lifetimeMs, maxBytes } = fields + if ( + typeof lifetimeMs !== 'number' || + !Number.isSafeInteger(lifetimeMs) || + lifetimeMs < 1 || + lifetimeMs > snapshotArchiveLimits.lifetimeMs || + typeof maxBytes !== 'number' || + !Number.isSafeInteger(maxBytes) || + maxBytes < snapshotArchiveLimits.headerCharge + 1 || + maxBytes > snapshotArchiveLimits.archiveBytes + ) + invalid() + return Object.freeze({ lifetimeMs, maxBytes }) +} + +export function snapshotArchiveReaderRequestId(request: Omit): string { + return syncTransferDigest( + new TextEncoder().encode( + JSON.stringify([ + 'wallet-snapshot-reader-request/1', + request.version, + request.nonce, + request.notAfter, + request.maxBytes + ]) + ) + ) +} + +/** The distinct version and digest domain prevent the legacy start API recreating a closed reader. */ +export function parseSnapshotArchiveReaderRequest(input: unknown): Readonly { + const names = ['version', 'nonce', 'notAfter', 'maxBytes', 'requestId'] + if ( + input === null || + typeof input !== 'object' || + Array.isArray(input) || + Reflect.ownKeys(input).length !== names.length + ) + invalid() + const values: Record = {} + for (const name of names) { + const field = Object.getOwnPropertyDescriptor(input, name) + if (field === undefined || !('value' in field) || !field.enumerable) invalid() + values[name] = field.value + } + const { nonce, notAfter, maxBytes } = values + if ( + values.version !== 2 || + typeof nonce !== 'string' || + !/^[0-9a-f]{64}$/.test(nonce) || + typeof notAfter !== 'number' || + !Number.isSafeInteger(notAfter) || + notAfter < 1 || + typeof maxBytes !== 'number' || + !Number.isSafeInteger(maxBytes) || + maxBytes < snapshotArchiveLimits.headerCharge + 1 || + maxBytes > snapshotArchiveLimits.archiveBytes + ) + invalid() + const fields = { version: 2 as const, nonce, notAfter, maxBytes } + const requestId = snapshotArchiveReaderRequestId(fields) + if (values.requestId !== requestId) invalid() + return Object.freeze({ ...fields, requestId }) +} + +export function validateSnapshotArchiveReaderRequest( + input: unknown, + now: number +): Readonly { + const request = parseSnapshotArchiveReaderRequest(input) + if ( + !Number.isSafeInteger(now) || + now < 0 || + request.notAfter <= now || + request.notAfter - now > snapshotArchiveLimits.lifetimeMs + ) + invalid() + return request +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveRequest.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveRequest.test.ts new file mode 100644 index 000000000..713ec4769 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveRequest.test.ts @@ -0,0 +1,125 @@ +import { createHash } from 'node:crypto' +import { + parseSnapshotArchiveRequest, + snapshotArchiveRequestId, + validateSnapshotArchiveRequest +} from './SnapshotArchiveRequest' + +const now = Date.UTC(2026, 8, 30) +function request() { + const value = { version: 1 as const, nonce: 'a'.repeat(64), notAfter: now + 300000, maxBytes: 32768 } + const requestId = createHash('sha256') + .update(JSON.stringify(['wallet-snapshot-request/1', 1, value.nonce, value.notAfter, value.maxBytes])) + .digest('hex') + return { ...value, requestId } +} + +test('binds every creation option and the immutable deadline to an independent digest', () => { + const value = request() + expect(snapshotArchiveRequestId(value)).toBe(value.requestId) + const parsed = validateSnapshotArchiveRequest(value, now) + expect(parsed).toEqual(value) + expect(parsed).not.toBe(value) + expect(Object.isFrozen(parsed)).toBe(true) + value.nonce = 'b'.repeat(64) + expect(parsed.nonce).toBe('a'.repeat(64)) +}) + +test.each(['nonce', 'notAfter', 'maxBytes'] as const)('a changed %s cannot reuse its previous request ID', key => { + const value = request() + const changed = key === 'nonce' ? 'b'.repeat(64) : value[key] + 1 + expect(() => validateSnapshotArchiveRequest({ ...value, [key]: changed }, now)).toThrow( + 'Invalid snapshot archive creation request' + ) +}) + +test.each([now + 300000, now + 300001])( + 'an expired request cannot reopen at %s after its receipt is collected', + clock => { + expect(() => validateSnapshotArchiveRequest(request(), clock)).toThrow('Invalid snapshot archive creation request') + } +) + +test.each([1, 3600000])('accepts a bounded remaining lifetime of %i milliseconds', lifetime => { + const value = { ...request(), notAfter: now + lifetime } + value.requestId = snapshotArchiveRequestId(value) + expect(validateSnapshotArchiveRequest(value, now).notAfter).toBe(value.notAfter) +}) + +test.each([4097, 32 * 1024 * 1024])('accepts a reservation of exactly %i bytes', maxBytes => { + const value = { ...request(), maxBytes } + value.requestId = snapshotArchiveRequestId(value) + expect(validateSnapshotArchiveRequest(value, now).maxBytes).toBe(maxBytes) +}) + +test.each([ + ['version', 2], + ['nonce', 'A'.repeat(64)], + ['nonce', 'a'.repeat(63)], + ['nonce', 'a'.repeat(65)], + ['nonce', 1], + ['notAfter', 0], + ['notAfter', NaN], + ['notAfter', Infinity], + ['notAfter', 0.5], + ['notAfter', 'timestamp'], + ['notAfter', now + 3600001], + ['maxBytes', 4096], + ['maxBytes', 32 * 1024 * 1024 + 1], + ['maxBytes', NaN], + ['maxBytes', '32768'], + ['maxBytes', 32768.5], + ['requestId', 'b'.repeat(64)] +])('refuses malformed request %s=%p even with a recomputed ID', (key, changed) => { + const value = { ...request(), [key]: changed } + if (key !== 'requestId') value.requestId = snapshotArchiveRequestId(value as ReturnType) + expect(() => validateSnapshotArchiveRequest(value, now)).toThrow('Invalid snapshot archive creation request') +}) + +test.each([null, undefined, 1, 'request', []])('refuses a non-record request %p', value => { + expect(() => parseSnapshotArchiveRequest(value)).toThrow('Invalid snapshot archive creation request') +}) + +test.each([NaN, Infinity, -1, 0.5])('refuses an invalid database clock %s', clock => { + expect(() => validateSnapshotArchiveRequest(request(), clock)).toThrow('Invalid snapshot archive creation request') +}) + +test('requires exact own data fields without invoking request accessors', () => { + const value = request() + expect(() => parseSnapshotArchiveRequest({ ...value, claimToken: 'secret' })).toThrow( + 'Invalid snapshot archive creation request' + ) + expect(() => parseSnapshotArchiveRequest({ ...value, [Symbol('extra')]: true })).toThrow( + 'Invalid snapshot archive creation request' + ) + let reads = 0 + Object.defineProperty(value, 'nonce', { + enumerable: true, + get: () => { + reads++ + return 'a'.repeat(64) + } + }) + expect(() => parseSnapshotArchiveRequest(value)).toThrow('Invalid snapshot archive creation request') + expect(reads).toBe(0) + const hidden = request() + Object.defineProperty(hidden, 'nonce', { value: hidden.nonce, enumerable: false }) + expect(() => parseSnapshotArchiveRequest(hidden)).toThrow('Invalid snapshot archive creation request') + const renamed: Record = { ...request(), replacement: 'a'.repeat(64) } + delete renamed.nonce + expect(() => parseSnapshotArchiveRequest(renamed)).toThrow('Invalid snapshot archive creation request') +}) + +test('a different version cannot reuse the valid version-one digest', () => { + expect(() => parseSnapshotArchiveRequest({ ...request(), version: 2 })).toThrow( + 'Invalid snapshot archive creation request' + ) +}) + +test('supports the exact epoch boundary while refusing negative verifier time', () => { + const value = { ...request(), notAfter: 1 } + value.requestId = snapshotArchiveRequestId(value) + expect(validateSnapshotArchiveRequest(value, 0).notAfter).toBe(1) + expect(() => validateSnapshotArchiveRequest(value, -1)).toThrow('Invalid snapshot archive creation request') + expect(() => validateSnapshotArchiveRequest(value, 1)).toThrow('Invalid snapshot archive creation request') +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveRequest.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveRequest.ts new file mode 100644 index 000000000..696cabb8a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveRequest.ts @@ -0,0 +1,85 @@ +import { syncTransferDigest } from '../../remoting/SyncTransfer' +import { snapshotArchiveLimits } from './SnapshotArchive' + +export interface SnapshotArchiveRequest { + version: 1 + nonce: string + notAfter: number + maxBytes: number + requestId: string +} + +export type SnapshotArchiveTerminalState = 'closed' | 'failed' | 'expired' | 'resource-limited' + +export interface SnapshotArchiveRequestReceipt { + version: 1 + requestId: string + expiresAt: number + state: 'building' | 'ready' | SnapshotArchiveTerminalState + archiveId?: string + digest?: string +} + +/** Internal admission ownership; never part of an RPC result or argument. */ +export interface SnapshotArchiveRequestOwner { + identityKey: string + requestId: string + claimToken: string +} + +function invalid(): never { + throw new TypeError('Invalid snapshot archive creation request') +} + +function fields(input: unknown): Omit { + if (input === null || typeof input !== 'object' || Array.isArray(input)) invalid() + const names = ['version', 'nonce', 'notAfter', 'maxBytes', 'requestId'] + if (Reflect.ownKeys(input).length !== names.length) invalid() + const data: Record = {} + for (const name of names) { + const field = Object.getOwnPropertyDescriptor(input, name) + if (field === undefined || !('value' in field) || !field.enumerable) invalid() + data[name] = field.value + } + if (data.version !== 1 || typeof data.nonce !== 'string' || !/^[0-9a-f]{64}$/.test(data.nonce)) invalid() + const notAfter = data.notAfter + const maxBytes = data.maxBytes + if (typeof notAfter !== 'number' || !Number.isSafeInteger(notAfter) || notAfter < 1) invalid() + if ( + typeof maxBytes !== 'number' || + !Number.isSafeInteger(maxBytes) || + maxBytes < snapshotArchiveLimits.headerCharge + 1 || + maxBytes > snapshotArchiveLimits.archiveBytes + ) + invalid() + const request = { version: 1 as const, nonce: data.nonce, notAfter, maxBytes } + if (data.requestId !== snapshotArchiveRequestId(request)) invalid() + return request +} + +/** A deadline is part of the request identity, so an expired ID cannot reopen. */ +export function snapshotArchiveRequestId(request: Omit): string { + return syncTransferDigest( + new TextEncoder().encode( + JSON.stringify(['wallet-snapshot-request/1', request.version, request.nonce, request.notAfter, request.maxBytes]) + ) + ) +} + +export function parseSnapshotArchiveRequest(input: unknown): Readonly { + const request = fields(input) + return Object.freeze({ ...request, requestId: snapshotArchiveRequestId(request) }) +} + +/** Validate against the database clock; callers obtain time from an authenticated offer. */ +export function validateSnapshotArchiveRequest(input: unknown, now: number): Readonly { + const request = parseSnapshotArchiveRequest(input) + if ( + !Number.isSafeInteger(now) || + now < 0 || + request.notAfter <= now || + request.notAfter - now > snapshotArchiveLimits.lifetimeMs + ) + invalid() + return request +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveServerLifecycle.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveServerLifecycle.test.ts new file mode 100644 index 000000000..757a2d753 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveServerLifecycle.test.ts @@ -0,0 +1,94 @@ +import { once } from 'node:events' +import { StorageServer } from '../../remoting/StorageServer' +import { StorageProvider } from '../../StorageProvider' +import { Wallet } from '../../../Wallet' +import { gate, snapshotHttpFixture } from '../../../../test/utils/snapshotArchiveHttpFixtures' + +afterEach(() => jest.restoreAllMocks()) + +function bareServer() { + return new StorageServer({} as StorageProvider, { + port: 0, + wallet: { chain: 'test' } as Wallet, + monetize: false, + logRpcRequests: false + }) +} + +test('unstarted servers close idempotently and a completed reusable server can start without a configured host', async () => { + const fixture = await snapshotHttpFixture() + try { + const empty = bareServer() + await empty.close() + expect(empty.close()).toBe(Reflect.get(empty, 'closing')) + const listening = jest.spyOn(console, 'log').mockImplementation(() => undefined) + empty.start() + await once(empty.server, 'listening') + expect(listening).toHaveBeenCalledWith('WalletStorageServer listening at http://localhost:0') + expect(empty.server.listening).toBe(true) + await empty.close() + empty.start() + await once(empty.server, 'listening') + await empty.close() + expect(empty.server.listening).toBe(false) + } finally { + await fixture.close() + } +}) + +test.each(['ERR_SERVER_NOT_RUNNING', 'EIO'])('close waits for the real callback outcome %s', async code => { + const server = bareServer() + let complete!: (error?: NodeJS.ErrnoException) => void + server.server = { + close: jest.fn((callback: typeof complete) => { + complete = callback + return server.server + }) + } + let settled = false + const pending = server.close() + const observed = pending.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + await new Promise(resolve => setImmediate(resolve)) + expect(settled).toBe(false) + const error = Object.assign(new Error('fixture listener close failure'), { code }) + complete(error) + if (code === 'ERR_SERVER_NOT_RUNNING') await expect(pending).resolves.toBeUndefined() + else { + await expect(pending).rejects.toBe(error) + expect(() => server.start()).toThrow('closing') + } + await observed +}) + +test('a capture cleanup rejection remains observable after the HTTP drain finishes', async () => { + const server = bareServer() + const http = gate() + const cleanup = new Error('fixture physical reader cleanup failed') + let finishHttp!: () => void + server.server = { + close: jest.fn((callback: () => void) => { + finishHttp = callback + http.resolve() + }) + } + Reflect.set(server, 'snapshotArchives', { close: () => Promise.reject(cleanup) }) + let settled = false + const pending = server.close() + const observed = pending.catch(() => { + settled = true + }) + await http.promise + await new Promise(resolve => setImmediate(resolve)) + expect(settled).toBe(false) + finishHttp() + await expect(pending).rejects.toBe(cleanup) + await observed + await expect(server.close()).rejects.toBe(cleanup) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts new file mode 100644 index 000000000..308805aa4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveService.property.test.ts @@ -0,0 +1,331 @@ +import fc from 'fast-check' +import { createHash } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { addSnapshotArchiveTables } from '../../schema/snapshotArchiveMigration' +import { addSnapshotArchiveRequestTable } from '../../schema/snapshotArchiveRequestMigration' +import { addSnapshotArchiveOwnerTable } from '../../schema/snapshotArchiveOwnerMigration' +import { addSnapshotArchiveGuardTable } from '../../schema/snapshotArchiveGuardMigration' +import { readGuardedSnapshotArchive, recoverSnapshotArchiveGuards } from './SnapshotArchiveGuard' +import * as ArchiveSql from './SnapshotArchiveSql' +import { SnapshotArchiveCleanupPendingError } from './SnapshotArchiveOwner' +import { KnexSnapshotArchiveStore } from './KnexSnapshotArchiveStore' +import { KnexSnapshotArchiveRequestStore } from './KnexSnapshotArchiveRequestStore' +import { snapshotArchiveTables, type SnapshotArchiveBinding, type SnapshotArchiveWriter } from './SnapshotArchive' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +fc.configureGlobal({ + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + ...(Number.isSafeInteger(requestedSeed) ? { seed: requestedSeed } : {}), + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +}) + +test('generated guarded-owner schedules keep quota through native close and fence each successor claim', async () => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-guard-property-')) + const config = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'guards.sqlite') }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + } + const db = knex(config) + const peer = knex(config) + const requests = new KnexSnapshotArchiveRequestStore(db, true, true) + const replacement = new KnexSnapshotArchiveRequestStore(peer, true, true) + const databaseNow = ArchiveSql.snapshotArchiveDatabaseNow + let expiredAt: number | undefined + const clock = jest + .spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow') + .mockImplementation(async k => expiredAt ?? (await databaseNow(k))) + const recover = async () => { + await recoverSnapshotArchiveGuards(peer, config) + await replacement.reap() + } + const gate = () => { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } + } + try { + await db.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(db) + await addSnapshotArchiveRequestTable(db) + await addSnapshotArchiveOwnerTable(db) + await addSnapshotArchiveGuardTable(db) + await db.schema.createTable('guard_values', table => { + table.integer('id').primary() + table.integer('value') + }) + await db('guard_values').insert({ id: 1, value: 0 }) + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 0, max: 1000000 }), + fc.boolean(), + fc.array(fc.constantFrom('recover', 'wrong-ack', 'write'), { minLength: 0, maxLength: 4 }), + async (value, expire, schedule) => { + const identityKey = '02' + '77'.repeat(32) + const offered = (await requests.offer(identityKey, { lifetimeMs: 300000, maxBytes: 32768 }))! + const { owner } = await requests.claimReader(identityKey, offered.request) + await db('guard_values').update({ value }) + const source = knex(config) + const opened = gate(), + stop = gate(), + closing = gate(), + finish = gate() + const destroy = source.client.destroyRawConnection.bind(source.client) + source.client.destroyRawConnection = async connection => { + closing.resolve() + await finish.promise + await destroy(connection) + } + const work = readGuardedSnapshotArchive(db, source, owner!, async trx => { + expect((await trx('guard_values').first()).value).toBe(value) + opened.resolve() + await stop.promise + expect((await trx('guard_values').first()).value).toBe(value) + }) + void work.catch(() => undefined) + try { + await opened.promise + if (expire) expiredAt = offered.request.notAfter + else await replacement.markReaderCancellation(identityKey, offered.request) + for (const action of schedule) { + if (action === 'recover') await recover() + else if (action === 'wrong-ack') await replacement.sourceClosed({ ...owner!, claimToken: 'incorrect' }) + else await peer('guard_values').update({ value: value + 1 }) + expect(Number((await db('snapshot_archive_capacity').first()).archives)).toBe(1) + } + stop.resolve() + await closing.promise + await recover() + expect(await db('snapshot_archive_owners').first()).toMatchObject(owner!) + expect(Number((await db('snapshot_archive_capacity').first()).archives)).toBe(1) + finish.resolve() + await work + await recover() + expect(await db('snapshot_archive_owners')).toHaveLength(0) + expect(Number((await db('snapshot_archive_capacity').first()).archives)).toBe(0) + await expect(requests.claimReader(identityKey, offered.request)).rejects.toThrow( + expire ? 'Invalid snapshot archive reader request' : 'unavailable' + ) + expect((await db('guard_values').first()).value).toBe(schedule.includes('write') ? value + 1 : value) + const next = (await requests.offer(identityKey, { lifetimeMs: 300000, maxBytes: 32768 }))! + const successor = await requests.claimReader(identityKey, next.request) + await replacement.sourceClosed(owner!) + expect(await db('snapshot_archive_owners').first()).toMatchObject(successor.owner!) + await replacement.markReaderCancellation(identityKey, next.request) + await recover() + expect(Number((await db('snapshot_archive_capacity').first()).archives)).toBe(0) + } finally { + stop.resolve() + finish.resolve() + await work.catch(() => undefined) + await source.destroy() + expiredAt = undefined + } + } + ) + ) + } finally { + clock.mockRestore() + await Promise.all([db.destroy(), peer.destroy()]) + await rm(directory, { recursive: true, force: true }) + } +}) + +test('generated remote cancellation schedules retain source quota until exact cleanup acknowledgement', async () => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-owner-property-')) + const open = () => + knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'owners.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const db = open() + const peer = open() + try { + await db.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(db) + await addSnapshotArchiveRequestTable(db) + await addSnapshotArchiveOwnerTable(db) + const requests = new KnexSnapshotArchiveRequestStore(db, true) + const replacement = new KnexSnapshotArchiveRequestStore(peer, true) + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 0, max: 13 }), + fc.array(fc.constantFrom('retry', 'reap', 'wrong-ack'), { minLength: 0, maxLength: 5 }), + fc.uint8Array({ minLength: 1, maxLength: 16 }), + async (pages, schedule, bytes) => { + const identityKey = '02' + bytes[0].toString(16).padStart(64, '0') + const issued = (await requests.offer(identityKey, { lifetimeMs: 300000, maxBytes: 32768 }))! + const { owner } = await requests.claimReader(identityKey, issued.request) + const date = new Date('2026-01-01T00:00:00.000Z') + let writer: SnapshotArchiveWriter | undefined + if (pages > 0) { + writer = await requests.begin(owner!, { + version: 1, + snapshotId: 'a'.repeat(64), + sourceSchema: 'owner-property-v1', + sourceStorage: { + created_at: date, + updated_at: date, + chain: 'test', + dbtype: 'SQLite', + storageIdentityKey: 'source', + storageName: '', + maxOutputScript: 1024 + }, + user: { created_at: date, updated_at: date, userId: 1, identityKey, activeStorage: 'source' } + }) + for (const [sequence, table] of snapshotArchiveTables.slice(0, pages).entries()) + await requests.append(owner!, writer, { sequence, table, rows: 0, done: true, bytes }) + } + await replacement.markReaderCancellation(identityKey, issued.request) + for (const action of schedule) { + if (action === 'retry') + expect((await requests.claimReader(identityKey, issued.request)).receipt.state).toBe('closed') + else if (action === 'reap') await replacement.reap() + else await replacement.sourceClosed({ ...owner!, claimToken: 'wrong-claim-token' }) + } + await expect(replacement.close(identityKey, issued.request.requestId)).rejects.toBeInstanceOf( + SnapshotArchiveCleanupPendingError + ) + expect(await db('snapshot_archive_owners')).toHaveLength(1) + expect(await db('snapshot_archive_pages')).toHaveLength(pages) + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + if (writer !== undefined) + await expect( + requests.append(owner!, writer, { + sequence: 0, + table: snapshotArchiveTables[0], + rows: 0, + done: true, + bytes + }) + ).rejects.toThrow('unavailable') + await requests.sourceClosed(owner!) + await replacement.close(identityKey, issued.request.requestId) + await replacement.close(identityKey, issued.request.requestId) + await expect(requests.claimReader(identityKey, issued.request)).rejects.toThrow('unavailable') + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await db('snapshot_archive_owners')).toHaveLength(0) + expect(await db('snapshot_archive_requests')).toHaveLength(0) + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + } + ) + ) + } finally { + await Promise.all([db.destroy(), peer.destroy()]) + await rm(directory, { recursive: true, force: true }) + } +}) + +test('generated creation/retry/close schedules preserve request identity, atomic publication and exact capacity', async () => { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-request-property-')) + const open = () => + knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'requests.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const db = open() + const peer = open() + try { + await db.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(db) + await addSnapshotArchiveRequestTable(db) + const requests = new KnexSnapshotArchiveRequestStore(db) + const replacement = new KnexSnapshotArchiveRequestStore(peer) + const archives = new KnexSnapshotArchiveStore(db) + await fc.assert( + fc.asyncProperty( + fc.integer({ min: 0, max: 2 }), + fc.integer({ min: 0, max: 13 }), + fc.integer({ min: 0, max: 3 }), + fc.uint8Array({ minLength: 1, maxLength: 16 }), + fc.constantFrom('failed' as const, 'closed' as const, 'resource-limited' as const), + async (stage, prefix, retries, bytes, terminalState) => { + const identityKey = '02' + bytes[0].toString(16).padStart(64, '0') + const other = '03' + '11'.repeat(32) + const notAfter = Date.now() + 300000 + const nonce = bytes[0].toString(16).padStart(64, '0') + const requestId = createHash('sha256') + .update(JSON.stringify(['wallet-snapshot-request/1', 1, nonce, notAfter, 32768])) + .digest('hex') + const request = { version: 1, nonce, notAfter, maxBytes: 32768, requestId } + const admitted = await requests.claim(identityKey, request) + expect(admitted.owner).toBeDefined() + expect(admitted.receipt.state).toBe('building') + for (let retry = 0; retry < retries; retry++) + expect(await replacement.claim(identityKey, request)).toEqual({ receipt: admitted.receipt }) + const date = new Date('2026-01-01T00:00:00.000Z') + const binding: SnapshotArchiveBinding = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceSchema: 'property-v1', + sourceStorage: { + created_at: date, + updated_at: date, + chain: 'test', + dbtype: 'SQLite', + storageIdentityKey: 'source', + storageName: '', + maxOutputScript: 1024 + }, + user: { created_at: date, updated_at: date, userId: 1, identityKey, activeStorage: 'source' } + } + if (stage > 0) { + const writer = await requests.begin(admitted.owner!, binding) + const pages = stage === 2 ? 13 : prefix + for (const [sequence, table] of snapshotArchiveTables.slice(0, pages).entries()) { + await archives.append(writer, { sequence, table, rows: bytes[0], done: true, bytes }) + } + await expect(archives.inspect(identityKey, writer.archiveId)).rejects.toThrow('unavailable') + if (stage === 2) { + const manifest = await requests.seal(admitted.owner!, writer) + const recovered = await replacement.claim(identityKey, request) + expect(recovered).toEqual({ + receipt: { ...admitted.receipt, state: 'ready', archiveId: writer.archiveId, digest: manifest.digest } + }) + expect(manifest.rows).toBe(13 * bytes[0]) + expect(manifest.expiresAt).toBe(notAfter) + expect( + (await new KnexSnapshotArchiveStore(peer).read(identityKey, writer.archiveId, bytes[0] % 13)).bytes + ).toEqual(bytes) + } + } + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 1, reservedBytes: 32768 }) + await replacement.close(other, requestId) + await expect(replacement.status(other, requestId)).rejects.toThrow('unavailable') + expect((await db('snapshot_archive_capacity').first()).archives).toBe(1) + await replacement.close(identityKey, requestId, terminalState) + await requests.close(identityKey, requestId) + const terminal = (await requests.claim(identityKey, request)).receipt + expect(terminal.state).toBe(terminalState) + expect(terminal.archiveId).toBeUndefined() + expect(await db('snapshot_archive_capacity').first()).toMatchObject({ archives: 0, reservedBytes: 0 }) + expect(await db('snapshot_archives')).toHaveLength(0) + expect(await db('snapshot_archive_pages')).toHaveLength(0) + const rows = await db('snapshot_archive_requests') + expect(rows).toHaveLength(1) + expect(Boolean(rows[0].released)).toBe(true) + // Each generated case starts with an independent empty request history. + await db('snapshot_archive_requests').delete() + } + ) + ) + } finally { + await Promise.all([db.destroy(), peer.destroy()]) + await rm(directory, { recursive: true, force: true }) + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveSql.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveSql.ts new file mode 100644 index 000000000..f4ff7478b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveSql.ts @@ -0,0 +1,28 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' + +export interface SnapshotArchiveCapacity { + archives: number + reservedBytes: number | string +} + +export async function snapshotArchiveDatabaseNow(k: Knex): Promise { + if (String(k.client.config.client).includes('mysql')) { + const [rows]: Array> = await k.raw( + 'SELECT FLOOR(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000) AS now' + ) + return Number(rows[0].now) + } + const rows: Array<{ now: number }> = await k.raw( + "SELECT CAST((julianday('now') - 2440587.5) * 86400000 AS INTEGER) AS now" + ) + return rows[0].now +} + +export async function lockSnapshotArchiveCapacity(k: Knex): Promise { + // SQLite reserves the writer; MySQL locks the shared auxiliary capacity row. + await k('snapshot_archive_capacity').where({ id: 1 }).update({ id: 1 }) + const row: SnapshotArchiveCapacity | undefined = await k('snapshot_archive_capacity').where({ id: 1 }).first() + if (row === undefined) throw new WERR_INVALID_OPERATION('Snapshot archive schema is unavailable') + return row +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransport.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransport.test.ts new file mode 100644 index 000000000..fd1cb8ba7 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransport.test.ts @@ -0,0 +1,114 @@ +import { SnapshotArchiveTransport } from './SnapshotArchiveTransport' +import { snapshotArchiveRequestId } from './SnapshotArchiveRequest' +import { snapshotArchiveReaderRequestId } from './SnapshotArchiveReaderRequest' +import { fixture, expected, now } from '../../../../test/utils/snapshotArchiveDirectoryFixtures' + +const fields = { version: 1 as const, nonce: 'a'.repeat(64), notAfter: now + 1000, maxBytes: 32768 } +const request = { ...fields, requestId: snapshotArchiveRequestId(fields) } +const source = () => { + const { directory, payloads, binding } = fixture() + const offer = { + version: 1 as const, + serverTime: now, + sourceStorageIdentityKey: expected.sourceStorageIdentityKey, + sourceSchema: binding.sourceSchema, + chain: 'test' as const + } + const ready = { + version: 1 as const, + requestId: request.requestId, + expiresAt: request.notAfter, + state: 'ready' as const, + archiveId: directory.archiveId, + digest: directory.digest + } + const rpc = jest.fn(async (method: string, params: unknown[], _signal?: AbortSignal): Promise => { + const input = params[0] as Record + if (method === 'getSnapshotArchiveOffer') return offer + if (method === 'startSnapshotArchive' || method === 'getSnapshotArchiveStatus') return ready + if (method === 'getSnapshotArchiveDirectory') return directory + if (method === 'readSnapshotArchivePage') { + const i = input.sequence as number + return { ...directory.receipts[i], bytes: payloads[i] } + } + return true + }) + const transport = new SnapshotArchiveTransport(rpc, expected.identityKey, expected.sourceStorageIdentityKey, 'test') + return { transport, rpc, ready, offer, directory, payloads } +} + +test.each(['readerOffer', 'admit', 'readerStatus', 'cancelRequest'] as const)( + 'an archive-only transport refuses %s before network I/O with the negotiated-capability error', + async method => { + const { transport, rpc } = source() + const readerFields = { ...fields, version: 2 as const } + const readerRequest = { ...readerFields, requestId: snapshotArchiveReaderRequestId(readerFields) } + const operation = + method === 'readerOffer' + ? transport.readerOffer({ lifetimeMs: 1000, maxBytes: 32768 }) + : transport[method](readerRequest) + await expect(operation).rejects.toThrow(new TypeError('Snapshot archive reader was not negotiated')) + expect(rpc).not.toHaveBeenCalled() + } +) + +test('auth transport binds exact immutable request/root/profile and forwards cancellation for each operation', async () => { + const { transport, rpc, ready, offer, payloads } = source() + const signal = new AbortController().signal + expect(await transport.offer(signal)).toEqual(offer) + expect(await transport.start(request, signal)).toEqual(ready) + expect(await transport.status(request, signal)).toEqual(ready) + const verified = await transport.directory(ready, offer, now, signal) + expect(await transport.page(verified, 12, signal)).toEqual(payloads[12]) + await transport.cancel(request.requestId, signal) + expect(rpc.mock.calls).toHaveLength(6) + for (const [, params, callSignal] of rpc.mock.calls) { + expect(params[0]).toMatchObject({ version: 1, identityKey: expected.identityKey }) + expect(callSignal).toBe(signal) + } + expect((rpc.mock.calls[1][1][0] as { request: unknown }).request).not.toBe(request) +}) + +test('declines malformed local arguments before network calls', async () => { + const { transport, rpc, ready, offer } = source() + expect(() => new SnapshotArchiveTransport(rpc, '', 'source', 'test')).toThrow() + await expect(transport.start({ ...request, maxBytes: 0 })).rejects.toThrow() + await expect(transport.status({ ...request, nonce: '' })).rejects.toThrow() + await expect(transport.cancel('')).rejects.toThrow() + for (const receipt of [ + { ...ready, state: 'building' as const }, + { ...ready, archiveId: undefined }, + { ...ready, digest: undefined } + ]) { + await expect(transport.directory(receipt, offer, now)).rejects.toThrow('not ready') + } + expect(rpc).not.toHaveBeenCalled() + const verified = await transport.directory(ready, offer, now) + rpc.mockClear() + for (const sequence of [-1, 0.1, 13, NaN]) + await expect(transport.page(verified, sequence)).rejects.toThrow('sequence') + expect(rpc).not.toHaveBeenCalled() +}) + +test('refuses changed expiry, root, profile, sequence and payload even when the server sends a success envelope', async () => { + const { transport, rpc, ready, offer, directory, payloads } = source() + await expect(transport.directory({ ...ready, expiresAt: ready.expiresAt + 1 }, offer, now)).rejects.toThrow( + 'expiry changed' + ) + await expect(transport.directory({ ...ready, digest: '0'.repeat(64) }, offer, now)).rejects.toThrow( + 'Invalid snapshot archive directory' + ) + const verified = await transport.directory(ready, offer, now) + rpc.mockResolvedValueOnce({ ...directory.receipts[1], bytes: payloads[1] }) + await expect(transport.page(verified, 0)).rejects.toThrow() + rpc.mockResolvedValueOnce({ ...directory.receipts[0], bytes: new Uint8Array([0]) }) + await expect(transport.page(verified, 0)).rejects.toThrow() + rpc.mockResolvedValueOnce(false) + await expect(transport.cancel(request.requestId)).rejects.toThrow('cancellation receipt') + rpc.mockResolvedValueOnce({ ...offer, sourceStorageIdentityKey: 'changed' }) + await expect(transport.offer()).rejects.toThrow() + rpc.mockResolvedValueOnce({ ...ready, requestId: '0'.repeat(64) }) + await expect(transport.start(request)).rejects.toThrow() + rpc.mockRejectedValueOnce(new Error('network failure')) + await expect(transport.status(request)).rejects.toThrow('network failure') +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransport.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransport.ts new file mode 100644 index 000000000..a28eb0922 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransport.ts @@ -0,0 +1,167 @@ +import { + validateSnapshotArchiveReaderOffer, + validateSnapshotArchiveReaderOptions, + type SnapshotArchiveReaderOptions, + type SnapshotArchiveReaderOffer +} from './SnapshotArchiveReaderOffer' +import { parseSnapshotArchiveReaderRequest, type SnapshotArchiveReaderRequest } from './SnapshotArchiveReaderRequest' +import { validateSnapshotArchiveAdmission, type SnapshotArchiveAdmission } from './SnapshotArchiveAdmission' +import { validateSnapshotArchiveCancellation } from './SnapshotArchiveCleanup' +import { + verifySnapshotArchiveDirectory, + verifySnapshotArchivePage, + type VerifiedSnapshotArchiveDirectory +} from './SnapshotArchiveDirectory' +import { + parseSnapshotArchiveRequest, + type SnapshotArchiveRequest, + type SnapshotArchiveRequestReceipt +} from './SnapshotArchiveRequest' +import { + parseSnapshotArchiveRpcInput, + validateSnapshotArchiveOffer, + validateSnapshotArchiveRequestReceipt, + type SnapshotArchiveMethod, + type SnapshotArchiveOffer +} from './SnapshotArchiveProtocol' + +export type SnapshotArchiveRpcCall = ( + method: SnapshotArchiveMethod, + params: unknown[], + signal?: AbortSignal +) => Promise + +/** Bounded authenticated transport; a verified row reader is a separate layer. */ +export class SnapshotArchiveTransport { + constructor( + private readonly rpc: SnapshotArchiveRpcCall, + private readonly identityKey: string, + private readonly sourceStorageIdentityKey: string, + private readonly chain: 'main' | 'test', + readonly supportsReader = false + ) { + parseSnapshotArchiveRpcInput('getSnapshotArchiveOffer', [{ version: 1, identityKey }]) + } + + private async call( + method: SnapshotArchiveMethod, + fields: Record, + signal?: AbortSignal + ): Promise { + const params = [{ version: 1, identityKey: this.identityKey, ...fields }] + parseSnapshotArchiveRpcInput(method, params) + return await this.rpc(method, params, signal) + } + + async offer(signal?: AbortSignal): Promise> { + return validateSnapshotArchiveOffer( + await this.call('getSnapshotArchiveOffer', {}, signal), + this.sourceStorageIdentityKey, + this.chain + ) + } + + async start(input: SnapshotArchiveRequest, signal?: AbortSignal): Promise> { + const request = parseSnapshotArchiveRequest(input) + return validateSnapshotArchiveRequestReceipt(await this.call('startSnapshotArchive', { request }, signal), request) + } + + async status(input: SnapshotArchiveRequest, signal?: AbortSignal): Promise> { + const request = parseSnapshotArchiveRequest(input) + return validateSnapshotArchiveRequestReceipt( + await this.call('getSnapshotArchiveStatus', { requestId: request.requestId }, signal), + request + ) + } + + private requireReader(): void { + if (!this.supportsReader) throw new TypeError('Snapshot archive reader was not negotiated') + } + + async readerOffer( + input: SnapshotArchiveReaderOptions, + signal?: AbortSignal + ): Promise> { + this.requireReader() + const options = validateSnapshotArchiveReaderOptions(input) + return validateSnapshotArchiveReaderOffer( + await this.call('getSnapshotArchiveReaderOffer', { options }, signal), + options, + this.sourceStorageIdentityKey, + this.chain + ) + } + + async admit(input: SnapshotArchiveReaderRequest, signal?: AbortSignal): Promise> { + this.requireReader() + const request = parseSnapshotArchiveReaderRequest(input) + return validateSnapshotArchiveAdmission(await this.call('admitSnapshotArchive', { request }, signal), request) + } + + async readerStatus( + input: SnapshotArchiveReaderRequest, + signal?: AbortSignal + ): Promise> { + this.requireReader() + const request = parseSnapshotArchiveReaderRequest(input) + return validateSnapshotArchiveRequestReceipt( + await this.call('getSnapshotArchiveStatus', { requestId: request.requestId }, signal), + request + ) + } + + async cancelRequest(input: SnapshotArchiveReaderRequest, signal?: AbortSignal): Promise { + this.requireReader() + const request = parseSnapshotArchiveReaderRequest(input) + validateSnapshotArchiveCancellation( + await this.call('cancelSnapshotArchiveRequest', { request }, signal), + request.requestId + ) + } + + async directory( + receipt: Readonly, + offer: Readonly, + now: number, + signal?: AbortSignal + ): Promise { + if (receipt.state !== 'ready' || receipt.archiveId === undefined || receipt.digest === undefined) + throw new TypeError('Snapshot archive is not ready') + const archiveId = receipt.archiveId + const digest = receipt.digest + const expiresAt = receipt.expiresAt + const sourceSchema = offer.sourceSchema + const value = await this.call('getSnapshotArchiveDirectory', { archiveId }, signal) + const verified = verifySnapshotArchiveDirectory( + value, + { + identityKey: this.identityKey, + chain: this.chain, + sourceStorageIdentityKey: this.sourceStorageIdentityKey, + archiveId, + digest, + sourceSchema + }, + now + ) + if (verified.manifest.expiresAt !== expiresAt) throw new TypeError('Snapshot archive expiry changed') + return verified + } + + async page(directory: VerifiedSnapshotArchiveDirectory, sequence: number, signal?: AbortSignal): Promise { + if (!Number.isSafeInteger(sequence) || sequence < 0 || sequence >= directory.receipts.length) + throw new TypeError('Invalid snapshot archive page sequence') + const receipt = directory.receipts[sequence] + const value = await this.call( + 'readSnapshotArchivePage', + { archiveId: directory.manifest.archiveId, sequence }, + signal + ) + return verifySnapshotArchivePage(value, receipt) + } + + async cancel(requestId: string, signal?: AbortSignal): Promise { + if ((await this.call('cancelSnapshotArchive', { requestId }, signal)) !== true) + throw new TypeError('Invalid snapshot archive cancellation receipt') + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.test.ts new file mode 100644 index 000000000..d39edfeaa --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.test.ts @@ -0,0 +1,68 @@ +import { + isSnapshotArchiveTransportFailure, + snapshotArchiveFetch, + SnapshotArchiveTransportFailure +} from './SnapshotArchiveTransportFailure' + +test('the dedicated native-fetch wrapper forwards successful responses and original request arguments', async () => { + const response = new Response('body', { headers: { 'x-fixture': 'unchanged' } }) + const native = jest.fn(async () => response) + const fetch = snapshotArchiveFetch(native) + const options = { method: 'POST', body: 'request', signal: new AbortController().signal } + expect(await fetch('http://127.0.0.1:1', options)).toBe(response) + expect(native).toHaveBeenCalledWith('http://127.0.0.1:1', options) + expect(response.bodyUsed).toBe(false) +}) + +test('only an actual native-fetch rejection is marked, preserving its original cause', async () => { + const cause = new Error('synthetic connection loss') + const native = jest.fn(async () => { + throw cause + }) + const error = await snapshotArchiveFetch(native)('http://127.0.0.1:1').catch(error => error) + expect(error).toBeInstanceOf(SnapshotArchiveTransportFailure) + expect(error.cause).toBe(cause) + expect(error.name).toBe('SnapshotArchiveTransportFailure') + expect(error.message).toBe('Snapshot archive transport failed before a response was available') + expect(isSnapshotArchiveTransportFailure(error)).toBe(true) + const sdkWrapper = new Error('SDK network wrapper') + Object.defineProperty(sdkWrapper, 'cause', { value: error }) + expect(isSnapshotArchiveTransportFailure(sdkWrapper)).toBe(true) + const secondWrapper = new Error('An unrelated failure') + Object.defineProperty(secondWrapper, 'cause', { value: sdkWrapper }) + expect(isSnapshotArchiveTransportFailure(secondWrapper)).toBe(false) +}) + +test('auth, framing, body bounds and arbitrary error names never authorize recovery', async () => { + for (const message of [ + 'Invalid signature', + 'Authenticated response frame exceeds the configured limit.', + 'SnapshotArchiveTransportFailure' + ]) { + const error = new Error(message) + error.name = 'SnapshotArchiveTransportFailure' + expect(isSnapshotArchiveTransportFailure(error)).toBe(false) + } + for (const value of [undefined, null, 1, 'network error', { cause: new SnapshotArchiveTransportFailure('loss') }]) + expect(isSnapshotArchiveTransportFailure(value)).toBe(false) + const getter = new Error('Unrelated') + Object.defineProperty(getter, 'cause', { + get: () => { + throw new Error('Getter must not run') + } + }) + expect(isSnapshotArchiveTransportFailure(getter)).toBe(false) + const bodyFailure = new Error('synthetic body read failure') + const response = new Response( + new ReadableStream({ + start(controller) { + controller.error(bodyFailure) + } + }) + ) + const native = jest.fn(async () => response) + const fetched = await snapshotArchiveFetch(native)('http://127.0.0.1:1') + const error = await fetched.text().catch(error => error) + expect(error).toBe(bodyFailure) + expect(isSnapshotArchiveTransportFailure(error)).toBe(false) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.ts new file mode 100644 index 000000000..969ba721a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.ts @@ -0,0 +1,38 @@ +/** Marker only for rejection of the native fetch promise, before response handling. */ +export class SnapshotArchiveTransportFailure extends Error { + override readonly cause: unknown + constructor(cause: unknown) { + super('Snapshot archive transport failed before a response was available') + this.name = 'SnapshotArchiveTransportFailure' + this.cause = cause + } +} + +/** Inject only into this client's dedicated AuthFetch; never wrap AuthFetch.fetch itself. */ +export function snapshotArchiveFetch(implementation: typeof fetch): typeof fetch { + return async (input, init) => { + try { + return await implementation(input, init) + } catch (error) { + throw new SnapshotArchiveTransportFailure(error) + } + } +} + +/** The SDK transport preserves a native-fetch error as its direct own data cause. */ +export function isSnapshotArchiveTransportFailure(error: unknown): boolean { + if (error instanceof SnapshotArchiveTransportFailure) return true + if (!(error instanceof Error)) return false + const cause = Object.getOwnPropertyDescriptor(error, 'cause') + return cause !== undefined && 'value' in cause && cause.value instanceof SnapshotArchiveTransportFailure +} + +/** Only immutable, idempotent requests may repeat after native connection loss. */ +export async function retrySnapshotArchiveOperation(operation: () => Promise): Promise { + try { + return await operation() + } catch (error) { + if (!isSnapshotArchiveTransportFailure(error)) throw error + return await operation() + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/captureKnexSnapshotArchive.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/captureKnexSnapshotArchive.ts new file mode 100644 index 000000000..975182041 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/captureKnexSnapshotArchive.ts @@ -0,0 +1,43 @@ +import type { Knex } from 'knex' +import type { Chain } from '../../../sdk/types' +import { WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import type { StorageKnex } from '../../StorageKnex' +import { KnexSnapshotArchiveStore } from './KnexSnapshotArchiveStore' +import type { SnapshotArchiveManifest } from './SnapshotArchive' +import { openKnexSnapshotArchiveSource } from './KnexSnapshotArchiveSource' +import { + captureSnapshotArchiveSource, + assertSnapshotArchiveCaptureActive, + type SnapshotArchiveCaptureOptions +} from './captureSnapshotArchiveSource' +export type { SnapshotArchiveCaptureOptions } from './captureSnapshotArchiveSource' + +/** + * Capture all thirteen raw standard tables from one retained SQL view. Pages + * use the existing binary frame, not BRC-38 canonical archive bytes. The reader + * must own a separate connection/pool from staging and foreground operations. + */ +export async function captureKnexSnapshotArchive( + reader: StorageKnex, + staging: Knex, + identityKey: string, + chain: Chain, + options: SnapshotArchiveCaptureOptions = {} +): Promise { + if (reader.knex === staging) { + throw new WERR_INVALID_PARAMETER('staging', 'a connection pool separate from the retained source reader') + } + if (chain !== 'main' && chain !== 'test') throw new WERR_INVALID_PARAMETER('chain', 'main or test') + // Detach caller-owned options before the first asynchronous boundary. + const { signal, lifetimeMs, maxBytes, onProgress } = options + assertSnapshotArchiveCaptureActive(signal) + const store = new KnexSnapshotArchiveStore(staging) + await store.reap() + const source = await openKnexSnapshotArchiveSource(reader, identityKey, { signal, lifetimeMs }) + return await captureSnapshotArchiveSource(source, store, identityKey, chain, { + signal, + lifetimeMs, + maxBytes, + onProgress + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/captureSnapshotArchiveSource.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/captureSnapshotArchiveSource.ts new file mode 100644 index 000000000..f55fa4afb --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/captureSnapshotArchiveSource.ts @@ -0,0 +1,121 @@ +import type { Chain } from '../../../sdk/types' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { encodeSyncTransfer } from '../../remoting/SyncTransfer' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import type { WalletSnapshotCursor, WalletSnapshotTable } from '../WalletReadSnapshot' +import type { KnexSnapshotArchiveStore } from './KnexSnapshotArchiveStore' +import { + snapshotArchiveLimits, + snapshotArchiveTables, + type SnapshotArchiveManifest, + type SnapshotArchiveWriter +} from './SnapshotArchive' +import type { SnapshotArchiveSource } from './KnexSnapshotArchiveSource' + +export interface SnapshotArchiveCaptureOptions { + signal?: AbortSignal + lifetimeMs?: number + maxBytes?: number + /** Acknowledged staging progress only; no source IDs, row contents or secrets. */ + onProgress?: (progress: { pages: number; rows: number; bytes: number }) => void +} + +export function assertSnapshotArchiveCaptureActive(signal: AbortSignal | undefined): void { + if (signal?.aborted === true) throw new WERR_INVALID_OPERATION('Snapshot archive capture was cancelled') +} + +function packedRows(rows: object[]): Array> { + return rows.map(row => + Object.fromEntries( + Object.entries(row).map(([key, value]) => { + if (value === null || typeof value !== 'object' || value instanceof Uint8Array) return [key, value] + // Retain SQL dates from other realms without asking the binary codec to + // classify their prototype. Standard wallet rows otherwise contain scalars. + const time = Date.prototype.getTime.call(value) + if (!Number.isFinite(time)) throw new WERR_INVALID_OPERATION('Snapshot source contains an invalid date') + return [key, new Date(time)] + }) + ) + ) +} + +export type SnapshotArchiveCaptureStore = Pick & { + /** The initiating error allows a durable request owner to retain a bounded failure classification. */ + close: (identityKey: string, archiveId: string, error?: unknown) => Promise +} + +/** Shared ordered capture; source.close must finish physical ownership cleanup before sealing. */ +export async function captureSnapshotArchiveSource( + source: SnapshotArchiveSource, + store: SnapshotArchiveCaptureStore, + identityKey: string, + chain: Chain, + options: SnapshotArchiveCaptureOptions = {} +): Promise { + const { signal, lifetimeMs, maxBytes, onProgress } = options + let writer: SnapshotArchiveWriter | undefined + let closed = false + try { + assertSnapshotArchiveCaptureActive(signal) + if (source.user.identityKey !== identityKey || source.sourceStorage.chain !== chain) { + throw new WERR_INVALID_OPERATION('Snapshot source does not match the requested profile and chain') + } + writer = await store.begin( + { + version: 1, + snapshotId: source.snapshotId, + sourceStorage: source.sourceStorage, + sourceSchema: source.sourceSchema, + user: source.user + }, + { lifetimeMs, maxBytes } + ) + await source.validateClosure() + const owner = writer + const progress = { pages: 0, rows: 0, bytes: 0 } + const captureTable = async (table: WalletSnapshotTable): Promise => { + let cursor: WalletSnapshotCursor | undefined + let done = false + function* pendingPages(): Generator { + while (!done) yield undefined + } + await runInSeries(pendingPages(), async () => { + assertSnapshotArchiveCaptureActive(signal) + if (progress.pages >= snapshotArchiveLimits.pages) { + throw new SnapshotResourceLimitError('Snapshot archive page limit exceeded') + } + // The conservative SQL charge bounds fetched payloads before encoding; + // framing/JSON escaping has a separate exact one-MiB admission below. + const page = await source.readPage(table, cursor, { maxRows: 128, maxBytes: 131072 }) + const bytes = encodeSyncTransfer({ version: 1, table, rows: packedRows(page.rows) }) + if (bytes.length > snapshotArchiveLimits.pageBytes) { + throw new SnapshotResourceLimitError('Snapshot archive encoded page limit exceeded') + } + assertSnapshotArchiveCaptureActive(signal) + await store.append(owner, { sequence: progress.pages, table, rows: page.rows.length, done: page.done, bytes }) + progress.pages++ + progress.rows += page.rows.length + progress.bytes += bytes.length + onProgress?.({ ...progress }) + assertSnapshotArchiveCaptureActive(signal) + cursor = page.cursor + done = page.done + }) + } + await runInSeries(snapshotArchiveTables, captureTable) + await source.close() + closed = true + assertSnapshotArchiveCaptureActive(signal) + const manifest = await store.seal(owner) + assertSnapshotArchiveCaptureActive(signal) + return manifest + } catch (error) { + // Preserve the initiating error; interrupted cleanup keeps its reservation + // and is recovered by reap rather than exposing a partial result. + if (writer !== undefined) await store.close(identityKey, writer.archiveId, error).catch(() => undefined) + throw error + } finally { + if (!closed) await source.close().catch(() => undefined) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/openRemoteSnapshot.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/openRemoteSnapshot.ts new file mode 100644 index 000000000..345d3abff --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/archive/openRemoteSnapshot.ts @@ -0,0 +1,114 @@ +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import type { WalletReadSnapshot, WalletReadSnapshotOptions } from '../WalletReadSnapshot' +import { snapshotArchiveLimits } from './SnapshotArchive' +import type { SnapshotArchiveRequestReceipt } from './SnapshotArchiveRequest' +import type { SnapshotArchiveReaderRequest } from './SnapshotArchiveReaderRequest' +import type { SnapshotArchiveTransport } from './SnapshotArchiveTransport' +import { RemoteSnapshotLease } from './RemoteSnapshotLease' +import { createRemoteSnapshotPageReader } from './RemoteSnapshotPageReader' + +/** Preserve both failures without depending on AggregateError in native runtimes. */ +export class RemoteSnapshotOpeningCleanupError extends Error { + constructor( + override readonly cause: unknown, + readonly cleanupError: unknown + ) { + super('Remote snapshot opening and cleanup failed') + this.name = 'RemoteSnapshotOpeningCleanupError' + } +} + +async function accepted( + transport: SnapshotArchiveTransport, + lease: RemoteSnapshotLease, + request: SnapshotArchiveReaderRequest +): Promise | undefined> { + // The exact durable tuple deduplicates admission; never renew or recapture. + const result = await lease.runIdempotent(signal => transport.admit(request, signal)) + return result.outcome === 'accepted' ? result.receipt : undefined +} + +async function waitForCapture( + transport: SnapshotArchiveTransport, + lease: RemoteSnapshotLease, + request: SnapshotArchiveReaderRequest, + receipt: Readonly +): Promise> { + function* intervals() { + let interval = 100 + while (receipt.state === 'building') { + yield interval + interval = Math.min(interval * 2, 1000) + } + } + await runInSeries(intervals(), async interval => { + await lease.wait(interval) + receipt = await lease.runIdempotent(signal => transport.readerStatus(request, signal)) + }) + return receipt +} + +/** Unsupported capacity may decline only before a source view is exposed. */ +export async function openRemoteSnapshot( + transport: SnapshotArchiveTransport, + options: WalletReadSnapshotOptions = {} +): Promise { + if (!transport.supportsReader) return undefined + const lease = new RemoteSnapshotLease(transport, options) + try { + const issued = await lease.run(signal => + transport.readerOffer({ lifetimeMs: lease.lifetimeMs, maxBytes: snapshotArchiveLimits.archiveBytes }, signal) + ) + if (issued.outcome === 'resource-limited') { + await lease.close() + return undefined + } + const { offer, request } = issued + lease.bindServerTime(offer.serverTime) + lease.own(request) + let receipt = await accepted(transport, lease, request) + if (receipt === undefined) { + await lease.close() + return undefined + } + receipt = await waitForCapture(transport, lease, request, receipt) + if (receipt.state === 'resource-limited') { + await lease.close() + return undefined + } + if (receipt.state !== 'ready') throw new WERR_INVALID_OPERATION(`Snapshot archive capture is ${receipt.state}`) + const directory = await lease.runIdempotent(signal => transport.directory(receipt, offer, lease.now(), signal)) + if (lease.now() >= directory.manifest.expiresAt) throw new WERR_INVALID_OPERATION('Remote snapshot expired') + const binding = directory.manifest.binding + const copyDates = (value: T): T => ({ + ...value, + created_at: new Date(value.created_at), + updated_at: new Date(value.updated_at) + }) + return Object.freeze({ + version: 1 as const, + snapshotId: binding.snapshotId, + get sourceStorage() { + return copyDates(binding.sourceStorage) + }, + get user() { + return copyDates(binding.user) + }, + expiresAt: lease.expiresAt, + get isOpen() { + return lease.isOpen + }, + closed: lease.closed, + close: () => lease.close(), + readPage: createRemoteSnapshotPageReader(transport, directory, lease) + }) + } catch (error) { + try { + await lease.close() + } catch (error_) { + throw new RemoteSnapshotOpeningCleanupError(error, error_) + } + throw error + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournal.property.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournal.property.test.ts new file mode 100644 index 000000000..7e9f3a7e5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournal.property.test.ts @@ -0,0 +1,1138 @@ +import fc from 'fast-check' +import { knex, type Knex } from 'knex' +import { + snapshotJournalRevision, + compareSnapshotJournalRevisions, + MAX_SNAPSHOT_JOURNAL_REVISION, + snapshotJournalRevision as rev +} from './SnapshotJournalRevision' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { + readSnapshotJournalMetadataPage, + snapshotJournalMetadataQuery, + type SnapshotJournalInterval, + type SnapshotJournalPosition +} from './SnapshotJournalPage' +import { + snapshotJournalSqliteObserverSql, + SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL +} from './SnapshotJournalSqliteObservers' +import { installSnapshotJournalSqliteClock } from './SnapshotJournalSqliteClock' +import { readGenerationIndexState } from '../../schema/snapshotSqliteIndexState' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { seedArchiveClosure } from '../../../../test/utils/snapshotArchiveFixtures' +import { + fixture, + value, + keyOf, + replace, + tables, + numeric, + profiles +} from '../../../../test/utils/snapshotSqliteFixtures' +import { installGeneration } from '../../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage } from '../../schema/snapshotSqliteIndexBootstrap' + +const MIN_PROPERTY_RUNS = 300 +const requestedRuns = Number.parseInt(process.env.FAST_CHECK_NUM_RUNS ?? '', 10) +const requestedSeed = Number.parseInt(process.env.FAST_CHECK_SEED ?? '', 10) +const replayPath = process.env.FAST_CHECK_PATH +const propertyParameters = { + numRuns: Number.isSafeInteger(requestedRuns) ? Math.max(MIN_PROPERTY_RUNS, requestedRuns) : MIN_PROPERTY_RUNS, + seed: Number.isSafeInteger(requestedSeed) ? requestedSeed : 3242026, + ...(replayPath !== undefined && replayPath !== '' ? { path: replayPath } : {}) +} +fc.configureGlobal(propertyParameters) + +function compareBigints(a: bigint, b: bigint): number { + if (a === b) return 0 + return a < b ? -1 : 1 +} + +describe('SnapshotJournalRevision', () => { + const boundaries = [ + '0', + '1', + '9', + '10', + '9007199254740991', + '9007199254740992', + '9007199254740993', + '9223372036854775806', + '9223372036854775807' + ] + test.each(boundaries)('preserves exact canonical revision %s', value => { + expect(snapshotJournalRevision(value)).toBe(value) + }) + + test.each([ + undefined, + null, + true, + 0, + 1, + 1n, + NaN, + Infinity, + [], + {}, + '', + '00', + '01', + '-1', + '+1', + ' 1', + '1 ', + '1\n', + '1.0', + '1e3', + '١', + '９', + '9223372036854775808', + '9999999999999999999', + '10000000000000000000', + '1'.repeat(10000) + ])('refuses a noncanonical or out-of-range revision %p', value => { + expect(() => snapshotJournalRevision(value)).toThrow(WERR_INVALID_OPERATION) + expect(() => snapshotJournalRevision(value)).toThrow('Invalid snapshot journal revision') + }) + + test('canonical ordering agrees with independent bigint arithmetic through the entire range', () => { + fc.assert( + fc.property( + fc.bigInt({ min: 0n, max: 9223372036854775807n }), + fc.bigInt({ min: 0n, max: 9223372036854775807n }), + (a, b) => { + const left = snapshotJournalRevision(a.toString()), + right = snapshotJournalRevision(b.toString()) + expect(compareSnapshotJournalRevisions(left, right)).toBe(compareBigints(a, b)) + expect(compareSnapshotJournalRevisions(left, left)).toBe(0) + expect(snapshotJournalRevision(JSON.parse(JSON.stringify(left)))).toBe(left) + } + ), + propertyParameters + ) + }) + + test('SQLite preserves exact decimal positions above 2^53 with typed range predicates', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + try { + await k.raw( + "CREATE TABLE revisions(revision INTEGER NOT NULL PRIMARY KEY CHECK(typeof(revision)='integer' AND revision>=0))" + ) + for (const revision of boundaries) await k('revisions').insert({ revision }) + const exact = await k('revisions').select(k.raw('CAST(revision AS TEXT) AS revision')).orderBy('revision') + // ORDER BY the integer column explicitly; the text alias otherwise changes ordering. + const sorted = await k('revisions') + .select(k.raw('CAST(revisions.revision AS TEXT) AS exactRevision')) + .orderBy('revisions.revision') + expect(sorted.map(row => snapshotJournalRevision(row.exactRevision))).toEqual(boundaries) + expect(exact).toHaveLength(boundaries.length) + const page = await k('revisions') + .select(k.raw('CAST(revision AS TEXT) AS exactRevision')) + .whereRaw('revision > CAST(? AS INTEGER)', ['9007199254740992']) + .orderBy('revision') + .limit(2) + expect(page.map(row => snapshotJournalRevision(row.exactRevision))).toEqual([ + '9007199254740993', + '9223372036854775806' + ]) + const query = k('revisions') + .whereRaw('revision > CAST(? AS INTEGER)', ['9007199254740992']) + .orderBy('revision') + .limit(2) + .toSQL() + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + query.sql, query.bindings) + expect(plan.some(row => row.detail.includes('SEARCH revisions USING INTEGER PRIMARY KEY'))).toBe(true) + expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false) + await expect( + k('revisions') + .where('revision', MAX_SNAPSHOT_JOURNAL_REVISION) + .update({ revision: k.raw('revision+1') }) + ).rejects.toThrow() + expect( + ( + await k('revisions') + .select(k.raw('CAST(revision AS TEXT) AS exactRevision')) + .whereRaw('revision = CAST(? AS INTEGER)', [MAX_SNAPSHOT_JOURNAL_REVISION]) + .first() + ).exactRevision + ).toBe(MAX_SNAPSHOT_JOURNAL_REVISION) + } finally { + await k.destroy() + } + }) +}) + +describe('SnapshotJournalPage', () => { + const base = 9007199254740992n + let interval: SnapshotJournalInterval + beforeEach(() => { + interval = { + stream: 'scope', + tableId: 12, + userId: 1, + floor: rev('0'), + low: rev(String(base)), + high: rev(String(base + 10n)), + limit: 2 + } + }) + + async function database(): Promise { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + await k.raw( + 'CREATE TABLE snapshot_journal_scope(tableId INTEGER,userId INTEGER,id1 INTEGER,id2 INTEGER,exactText TEXT COLLATE BINARY,revision INTEGER,present INTEGER,PRIMARY KEY(tableId,userId,id1,id2,exactText))' + ) + await k.raw( + 'CREATE INDEX snapshot_journal_scope_page ON snapshot_journal_scope(userId,tableId,revision,id1,id2,exactText)' + ) + await k.raw( + 'CREATE TABLE snapshot_journal_physical(tableId INTEGER,id1 INTEGER,id2 INTEGER,exactText TEXT COLLATE BINARY,revision INTEGER,generation INTEGER,present INTEGER,PRIMARY KEY(tableId,id1,id2,exactText))' + ) + await k.raw( + 'CREATE INDEX snapshot_journal_physical_page ON snapshot_journal_physical(tableId,revision,id1,id2,exactText)' + ) + return k + } + + const compare = (a: SnapshotJournalPosition, b: SnapshotJournalPosition) => + compareBigints(BigInt(a.revision), BigInt(b.revision)) || + a.id1 - b.id1 || + a.id2 - b.id2 || + Buffer.compare(Buffer.from(a.exactText), Buffer.from(b.exactText)) + + test('real SQLite pages preserve 64-bit revisions, byte-exact ties and fixed upper bounds', async () => { + const k = await database() + try { + const rows = ['a', 'A', 'a ', 'a\0b', 'é', '中', '😀'].map((exactText, i) => ({ + tableId: 12, + userId: 1, + id1: i < 5 ? 1 : 2, + id2: 0, + exactText, + revision: rev(String(base + BigInt(i < 4 ? 1 : 2))), + present: i % 2 + })) + await k('snapshot_journal_scope').insert(rows) + await k('snapshot_journal_scope').insert({ ...rows[0], userId: 2 }) + await k('snapshot_journal_scope').insert({ ...rows[0], tableId: 3 }) + await k('snapshot_journal_scope').insert({ + ...rows[0], + id1: 999, + revision: String(base + 11n) + }) + const actual = [] + let after: SnapshotJournalPosition | undefined + for (let pages = 0; pages < 10; pages++) { + const page = await readSnapshotJournalMetadataPage(k, { ...interval, after }) + expect(page.examined).toBe(page.rows.length) + expect(page.examined).toBeLessThanOrEqual(2) + actual.push(...page.rows) + if (page.complete) break + expect(page.after).toBeDefined() + after = page.after + } + expect(actual).toEqual( + rows.sort(compare).map(({ tableId: _table, userId: _user, present, ...row }) => ({ + ...row, + present: present === 1 + })) + ) + const query = snapshotJournalMetadataQuery(k, { ...interval, after }).toSQL() + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + query.sql, query.bindings) + expect(plan.some(row => row.detail.includes('SEARCH j USING INDEX snapshot_journal_scope_page'))).toBe(true) + expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false) + } finally { + await k.destroy() + } + }) + + test('physical global pages charge every examined row and preserve exact insertion generations', async () => { + const k = await database() + try { + await k('snapshot_journal_physical').insert( + Array.from({ length: 5 }, (_, i) => ({ + tableId: 8, + id1: i + 1, + id2: 0, + exactText: '', + revision: String(base + BigInt(i) + 1n), + generation: String(base + 1n), + present: i % 2 + })) + ) + const page = await readSnapshotJournalMetadataPage(k, { + ...interval, + stream: 'physical', + tableId: 8 + }) + expect(page.examined).toBe(2) + expect(page.complete).toBe(false) + expect(page.rows.map(row => row.generation)).toEqual([String(base + 1n), String(base + 1n)]) + expect(page.rows.map(row => row.present)).toEqual([false, true]) + expect(await k('snapshot_journal_scope')).toEqual([]) + } finally { + await k.destroy() + } + }) + + test('late SQLite pages seek the complete composite cursor within a large tied revision', async () => { + const k = await database() + try { + for (let first = 1; first <= 10000; first += 200) { + await k('snapshot_journal_scope').insert( + Array.from({ length: 200 }, (_, i) => ({ + tableId: 12, + userId: 1, + id1: first + i, + id2: 0, + exactText: '', + revision: String(base + 1n), + present: 1 + })) + ) + } + const request = { + ...interval, + after: { revision: rev(String(base + 1n)), id1: 9900, id2: 0, exactText: '' } + } + expect((await readSnapshotJournalMetadataPage(k, request)).rows.map(row => row.id1)).toEqual([9901, 9902]) + const sql = snapshotJournalMetadataQuery(k, request).toSQL() + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + sql.sql, sql.bindings) + expect(plan.some(row => row.detail.includes('(revision,id1,id2,exactText)>(?,?,?,?)'))).toBe(true) + expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false) + const program: Array<{ opcode: string; p4: string | null }> = await k.raw('EXPLAIN ' + sql.sql, sql.bindings) + expect(program.some(step => ['SeekGE', 'SeekGT'].includes(step.opcode) && step.p4 === '6')).toBe(true) + } finally { + await k.destroy() + } + }) + + test('quiescent intervals issue no journal query, and stale floors refuse before I/O', async () => { + const k = await database() + const seen = jest.fn() + k.on('query', seen) + try { + expect(await readSnapshotJournalMetadataPage(k, { ...interval, high: interval.low })).toEqual({ + rows: [], + examined: 0, + complete: true + }) + await expect(readSnapshotJournalMetadataPage(k, { ...interval, floor: interval.high })).rejects.toThrow( + 'Invalid snapshot journal interval or metadata' + ) + expect(seen).not.toHaveBeenCalled() + } finally { + await k.destroy() + } + }) + + test.each([ + { limit: 0 }, + { limit: 257 }, + { limit: 1.5 }, + { limit: NaN }, + { userId: 0 }, + { userId: Number.MAX_SAFE_INTEGER + 1 }, + { tableId: 13 }, + { tableId: -1 }, + { stream: 'physical', tableId: 12 }, + { stream: 'unknown' }, + { high: '0' }, + { low: '01' }, + { high: '9223372036854775808' }, + { after: { revision: String(base), id1: 1, id2: 0, exactText: '' } }, + { after: { revision: String(base + 11n), id1: 1, id2: 0, exactText: '' } }, + { after: { revision: String(base + 10n), id1: 0, id2: 0, exactText: '' } }, + { after: { revision: String(base + 10n), id1: 1, id2: -1, exactText: '' } }, + { after: { revision: String(base + 10n), id1: 1, id2: 0, exactText: '😀'.repeat(101) } }, + { after: { revision: String(base + 10n), id1: 1, id2: 0, exactText: '\ud800' } } + ])('malformed request refuses before SQL: %p', async change => { + const k = await database() + const seen = jest.fn() + k.on('query', seen) + try { + await expect( + readSnapshotJournalMetadataPage(k, { ...interval, ...change } as SnapshotJournalInterval) + ).rejects.toThrow() + expect(seen).not.toHaveBeenCalled() + } finally { + await k.destroy() + } + }) + + test.each([ + { present: 2 }, + { present: 'true' }, + { id1: 0 }, + { id2: -1 }, + { exactText: '😀'.repeat(101) }, + { exactText: Buffer.from([0xff]) }, + { generation: '0' }, + { generation: String(base + 2n) } + ])('invalid native metadata is never emitted: %p', async change => { + const k = await database() + try { + await k('snapshot_journal_physical').insert({ + tableId: 8, + id1: 1, + id2: 0, + exactText: '', + revision: String(base + 1n), + generation: String(base + 1n), + present: 1, + ...change + }) + await expect( + readSnapshotJournalMetadataPage(k, { ...interval, stream: 'physical', tableId: 8 }) + ).rejects.toThrow() + } finally { + await k.destroy() + } + }) + + test('generated bounded pages equal an independent exact integer and byte-order oracle', async () => { + const k = await database() + try { + await fc.assert( + fc.asyncProperty( + fc.uniqueArray( + fc.record({ + userId: fc.integer({ min: 1, max: 2 }), + tableId: fc.constantFrom(3, 12), + id1: fc.integer({ min: 1, max: 8 }), + id2: fc.integer({ min: 0, max: 3 }), + exactText: fc.constantFrom('', 'a', 'A', 'a ', 'é', '😀', 'a\0b'), + offset: fc.integer({ min: 0, max: 11 }), + present: fc.boolean() + }), + { + maxLength: 40, + selector: row => JSON.stringify([row.userId, row.tableId, row.id1, row.id2, row.exactText]) + } + ), + fc.integer({ min: 1, max: 6 }), + async (records, limit) => { + await k('snapshot_journal_scope').delete() + const rows = records.map(({ offset, present, ...row }) => ({ + ...row, + revision: rev(String(base + BigInt(offset))), + present: Number(present) + })) + if (rows.length) await k('snapshot_journal_scope').insert(rows) + const actual = [] + let after: SnapshotJournalPosition | undefined + for (let pageIndex = 0; pageIndex <= rows.length; pageIndex++) { + const page = await readSnapshotJournalMetadataPage(k, { ...interval, limit, after }) + expect(page.examined).toBeLessThanOrEqual(limit) + actual.push(...page.rows) + if (page.complete) break + after = page.after + } + expect(actual).toEqual( + rows + .filter( + row => + row.tableId === 12 && + row.userId === 1 && + BigInt(row.revision) > base && + BigInt(row.revision) <= base + 10n + ) + .sort(compare) + .map(({ tableId: _table, userId: _user, present, ...row }) => ({ + ...row, + present: present === 1 + })) + ) + } + ), + propertyParameters + ) + } finally { + await k.destroy() + } + }) + + test.each(['oversized response', 'at lower bound', 'above upper bound', 'out of order', 'null text'])( + 'driver response integrity rejects %s before returning a page', + async kind => { + const k = await database() + const original = k.client.processResponse.bind(k.client) + const row = { revisionText: String(base + 1n), id1: 1, id2: 0, exactText: '', present: 1 } + const rows: Array> = [row] + if (kind === 'oversized response') rows.push({ ...row, id1: 2 }, { ...row, id1: 3 }) + if (kind === 'at lower bound') row.revisionText = interval.low + if (kind === 'above upper bound') row.revisionText = String(base + 11n) + if (kind === 'out of order') rows.push({ ...row }) + if (kind === 'null text') rows[0] = { ...row, exactText: null } + const spy = jest.spyOn(k.client, 'processResponse').mockImplementation((...args: unknown[]) => { + const query = args[0] as { sql: string } + return query.sql.includes('AS `j` INDEXED BY') ? rows : original(...args) + }) + try { + await expect(readSnapshotJournalMetadataPage(k, interval)).rejects.toThrow( + 'Invalid snapshot journal interval or metadata' + ) + expect(spy).toHaveBeenCalled() + } finally { + spy.mockRestore() + await k.destroy() + } + } + ) + test('cursor text must be a string before any SQL is issued', async () => { + const k = await database(), + seen = jest.fn() + k.on('query', seen) + try { + await expect( + readSnapshotJournalMetadataPage(k, { + ...interval, + after: { revision: interval.high, id1: 1, id2: 0, exactText: Buffer.from('a') } + } as unknown as SnapshotJournalInterval) + ).rejects.toThrow('Invalid snapshot journal interval or metadata') + expect(seen).not.toHaveBeenCalled() + } finally { + await k.destroy() + } + }) + test('page query refuses unsupported drivers before SQL', async () => { + const k = await database(), + seen = jest.fn() + k.client.config.client = 'unsupported' + k.on('query', seen) + try { + await expect(readSnapshotJournalMetadataPage(k, interval)).rejects.toThrow( + 'Unsupported snapshot journal SQL driver' + ) + expect(seen).not.toHaveBeenCalled() + } finally { + await k.destroy() + } + }) +}) + +describe('SnapshotJournalSqliteObservers', () => { + // Internal journal qualification; reader advertisement remains a separate gate. + + const q = (name: string) => '"' + name.replaceAll('"', '""') + '"' + const scopeKey = 'tableId,userId,id1,id2,exactText' + const physicalKey = 'tableId,id1,id2,exactText' + const tuple = (table: string, prefix: string) => { + const key = numeric.find(([name]) => name === table)?.[1] + if (key) return [prefix + '.' + q(key), '0', "''"] + if (table === 'tx_labels_map') return [prefix + '.txLabelId', prefix + '.transactionId', "''"] + if (table === 'output_tags_map') return [prefix + '.outputTagId', prefix + '.outputId', "''"] + return [prefix + '.certificateId', '0', prefix + '.fieldName'] + } + function physicalIdentity(tableId: number, table: string, row: Record) { + const key = numeric[tableId]?.[1] + let id1 = row.certificateId, + id2: unknown = 0 + if (key) id1 = row[key] + else if (table === 'tx_labels_map') { + id1 = row.txLabelId + id2 = row.transactionId + } else if (table === 'output_tags_map') { + id1 = row.outputTagId + id2 = row.outputId + } + return { tableId, id1, id2, exactText: table === 'certificate_fields' ? row.fieldName : '' } + } + async function installCandidate(k: Knex, reverse: boolean) { + const definitions = await snapshotJournalSqliteObserverSql(k) + await installSnapshotJournalSqliteClock(k, snapshotJournalRevision('1000000000')) + for (const ddl of SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL) await k.raw(ddl) + const prior: Array<{ name: string; sql: string }> = await k('sqlite_master') + .where('type', 'trigger') + .whereIn('tbl_name', tables) + .select('name', 'sql') + for (const definition of definitions) await k.raw(definition) + if (reverse) + for (const trigger of prior) { + await k.raw('DROP TRIGGER ??', [trigger.name]) + await k.raw(trigger.sql) + } + } + async function expected(k: Knex) { + const selections = profiles.map( + ([table, key], id) => `SELECT ${id} tableId,userId,${q(key)} id1,0 id2,'' exactText FROM ${q(table)}` + ) + for (const [id, table, left, leftKey, right, rightKey] of [ + [10, 'tx_labels_map', 'tx_labels', 'txLabelId', 'transactions', 'transactionId'], + [11, 'output_tags_map', 'output_tags', 'outputTagId', 'outputs', 'outputId'] + ]) + for (const [parent, key] of [ + [left, leftKey], + [right, rightKey] + ]) + selections.push( + `SELECT ${id},p.userId,m.${leftKey},m.${rightKey},'' FROM ${table} m JOIN ${parent} p ON m.${key}=p.${key}` + ) + selections.push( + 'SELECT 12,userId,certificateId,0,fieldName FROM certificate_fields', + 'SELECT 12,c.userId,f.certificateId,0,f.fieldName FROM certificate_fields f JOIN certificates c ON c.certificateId=f.certificateId', + "SELECT 9,t.userId,r.provenTxReqId,0,'' FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid", + "SELECT 8,t.userId,p.provenTxId,0,'' FROM transactions t JOIN proven_txs p ON p.provenTxId=t.provenTxId", + "SELECT 8,t.userId,p.provenTxId,0,'' FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid JOIN proven_txs p ON p.provenTxId=r.provenTxId" + ) + return await k.raw( + 'SELECT * FROM (' + selections.join(' UNION ') + ') ORDER BY tableId,userId,id1,id2,exactText COLLATE BINARY' + ) + } + async function exact(k: Knex) { + expect( + await k('snapshot_journal_scope').where('present', 1).select(scopeKey.split(',')).orderBy(scopeKey.split(',')) + ).toEqual(await expected(k)) + for (const [tableId, table] of tables.entries()) { + const keys = tuple(table, 's') + const rows = await k.raw( + `SELECT s.*,g.generation AS candidateGeneration,g.present AS candidatePresent FROM ${q(table)} s LEFT JOIN snapshot_journal_physical g ON g.tableId=${tableId} AND g.id1=${keys[0]} AND g.id2=${keys[1]} AND g.exactText COLLATE BINARY=${keys[2]} COLLATE BINARY` + ) + for (const row of rows) { + expect(row.candidatePresent).toBe(1) + expect(row.candidateGeneration).toBeGreaterThan(0) + } + } + } + + test.each( + ['BINARY', 'NOCASE', 'RTRIM'].flatMap(collation => + [false, true].flatMap(reverse => [false, true].map(recursive => ({ collation, reverse, recursive }))) + ) + )( + 'all thirteen source/scope observers, collation=$collation order=$reverse recursion=$recursive', + async ({ collation, reverse, recursive }) => { + const k = await fixture(collation, recursive, false) + try { + for (const table of tables) + await k.schema.alterTable(table, t => { + void t.text('payload') + void t.text('quoted"payload') + void t.integer('updated_at').notNullable().defaultTo(0) + }) + const plan = await installGeneration(k) + for (let n = 0; n < 100; n++) if ((await copyGenerationPage(k, plan)).complete) break + await installCandidate(k, reverse) + for (const table of tables) + for (const id of [1, 2]) + await k(table).insert({ + ...value(table, id, id, id), + payload: 'original' + }) + await exact(k) + const before = await k('snapshot_journal_physical').orderBy(physicalKey.split(',')) + const scopeBefore = await k('snapshot_journal_scope').orderBy(scopeKey.split(',')) + for (const table of tables) await k(table).update({ payload: 'same-timestamp update' }) + await exact(k) + const after = await k('snapshot_journal_physical').orderBy(physicalKey.split(',')) + expect(after.map(r => r.generation)).toEqual(before.map(r => r.generation)) + after.forEach((r, i) => expect(r.revision).toBeGreaterThan(before[i].revision)) + const scopeAfter = await k('snapshot_journal_scope').orderBy(scopeKey.split(',')) + expect(scopeAfter).toHaveLength(scopeBefore.length) + scopeAfter.forEach((row, i) => { + if (row.tableId !== 8 && row.tableId !== 9) expect(row.revision).toBeGreaterThan(scopeBefore[i].revision) + }) + for (const table of tables) await k(table).update({ payload: 'same-timestamp update' }) + expect(await k('snapshot_journal_physical').orderBy(physicalKey.split(','))).toEqual(after) + await k.raw( + 'CREATE TRIGGER candidate_nested_owner AFTER INSERT ON transactions WHEN NEW.userId=2 BEGIN UPDATE transactions SET userId=3 WHERE transactionId=NEW.transactionId; END' + ) + await replace(k, 'transactions', value('transactions', 1, 2, 2)) + await exact(k) + await k.raw( + "CREATE TRIGGER candidate_nested_reinsert AFTER UPDATE ON outputs WHEN NEW.payload='replace-inside' BEGIN DELETE FROM outputs WHERE outputId=NEW.outputId; INSERT INTO outputs(outputId,userId,transactionId,vout,payload,updated_at) VALUES(NEW.outputId,NEW.userId,NEW.transactionId,NEW.vout,'nested-new',NEW.updated_at); END" + ) + const oldGeneration = ( + await k('snapshot_journal_physical').where({ tableId: 1, id1: 1, id2: 0, exactText: '' }).first() + ).generation + await k('outputs').where('outputId', 1).update({ payload: 'replace-inside' }) + await exact(k) + expect( + (await k('snapshot_journal_physical').where({ tableId: 1, id1: 1, id2: 0, exactText: '' }).first()).generation + ).toBeGreaterThan(oldGeneration) + const field = await k('certificate_fields').where('certificateId', 1).first() + const renamed = collation === 'RTRIM' ? field.fieldName + ' ' : field.fieldName.toUpperCase() + await k('certificate_fields') + .where({ certificateId: 1, fieldName: field.fieldName }) + .update({ fieldName: renamed }) + await exact(k) + expect( + await k('snapshot_journal_scope').where({ + tableId: 12, + id1: 1, + exactText: field.fieldName, + present: 1 + }) + ).toEqual([]) + const originalLabel = await k('tx_labels').where('txLabelId', 1).first() + const labelGeneration = ( + await k('snapshot_journal_physical').where({ tableId: 3, id1: 1, id2: 0, exactText: '' }).first() + ).generation + await k('tx_labels').where('txLabelId', 1).update({ userId: 4 }) + await exact(k) + expect( + (await k('snapshot_journal_physical').where({ tableId: 3, id1: 1, id2: 0, exactText: '' }).first()).generation + ).toBe(labelGeneration) + expect( + ( + await k('snapshot_journal_scope') + .where({ + tableId: 3, + userId: originalLabel.userId, + id1: 1, + id2: 0, + exactText: '' + }) + .first() + ).present + ).toBe(0) + await k('tx_labels').where('txLabelId', 1).delete() + await exact(k) + await k('tx_labels').insert(originalLabel) + await exact(k) + expect( + (await k('snapshot_journal_physical').where({ tableId: 3, id1: 1, id2: 0, exactText: '' }).first()).generation + ).toBeGreaterThan(labelGeneration) + const globalScopes = await k('snapshot_journal_scope').where('tableId', 8).orderBy(scopeKey.split(',')) + const proofBefore = await k('snapshot_journal_physical') + .where({ tableId: 8, id1: 1, id2: 0, exactText: '' }) + .first() + await k('proven_txs').where('provenTxId', 1).update({ payload: 'new global payload with equal timestamp' }) + const proofAfter = await k('snapshot_journal_physical') + .where({ tableId: 8, id1: 1, id2: 0, exactText: '' }) + .first() + expect(proofAfter.generation).toBe(proofBefore.generation) + expect(proofAfter.revision).toBeGreaterThan(proofBefore.revision) + expect(await k('snapshot_journal_scope').where('tableId', 8).orderBy(scopeKey.split(','))).toEqual(globalScopes) + await exact(k) + await fc.assert( + fc.asyncProperty( + fc.array( + fc.record({ + table: fc.integer({ min: 0, max: 12 }), + kind: fc.integer({ min: 0, max: 3 }), + id: fc.integer({ min: 1, max: 5 }), + other: fc.integer({ min: 1, max: 5 }), + user: fc.integer({ min: 1, max: 3 }) + }), + { minLength: 1, maxLength: 10 } + ), + async ops => { + for (const op of ops) { + const table = tables[op.table] + if (op.kind === 0) await replace(k, table, value(table, op.id, op.other, op.user)) + else if (op.kind === 1) + await k(table) + .where(keyOf(table, op.id, op.other)) + .delete() + else { + const query = k(table) + .where(keyOf(table, op.id, op.other)) + .update(value(table, op.other, op.id, op.user)) + .toSQL() + await k.raw( + query.sql.replace(/^update/i, op.kind === 2 ? 'UPDATE OR REPLACE' : 'UPDATE OR IGNORE'), + query.bindings + ) + } + await exact(k) + } + } + ), + propertyParameters + ) + } finally { + await k.destroy() + } + }, + 60000 + ) + + test.each([false, true].flatMap(reverse => [false, true].map(recursive => ({ reverse, recursive }))))( + 'actual migrated schema and independent WAL view, order=$reverse recursion=$recursive', + async ({ reverse, recursive }) => { + const directory = await mkdtemp(join(tmpdir(), 'ts569-journal-actual-')) + const filename = join(directory, 'wallet.sqlite') + const k = knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const reader = knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const source = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: k + }) + let view: Knex.Transaction | undefined + try { + await k.raw('PRAGMA journal_mode=WAL') + await source.migrate('candidate journal fixture', 'synthetic-candidate-journal') + await source.makeAvailable() + await k.raw('PRAGMA recursive_triggers=' + Number(recursive)) + const users = [] + for (const identity of ['02' + '11'.repeat(32), '03' + '22'.repeat(32)]) + users.push((await source.findOrInsertUser(identity)).user.userId) + await installCandidate(k, reverse) + expect(await readGenerationIndexState(k)).toBe('v2') + await seedArchiveClosure(source, users[0], users[1]) + await exact(k) + const physicalBefore = await k('snapshot_journal_physical').orderBy(physicalKey.split(',')) + const scopeBefore = await k('snapshot_journal_scope').orderBy(scopeKey.split(',')) + const highBefore = (await k('snapshot_journal_clock').first()).revision + view = await reader.transaction() + expect((await view('snapshot_journal_clock').first()).revision).toBe(highBefore) + for (const table of tables) { + const timestamps = await k(table).select('updated_at') + await k(table).update({ created_at: '2026-01-01T00:00:00.001Z' }) + expect(await k(table).select('updated_at')).toEqual(timestamps) + } + await exact(k) + expect(await view('snapshot_journal_physical').orderBy(physicalKey.split(','))).toEqual(physicalBefore) + expect(await view('snapshot_journal_scope').orderBy(scopeKey.split(','))).toEqual(scopeBefore) + const physicalAfter = await k('snapshot_journal_physical').orderBy(physicalKey.split(',')) + expect(physicalAfter.map(r => r.generation)).toEqual(physicalBefore.map(r => r.generation)) + expect( + new Set((await k('snapshot_journal_physical').where('revision', '>', highBefore)).map(r => r.tableId)) + ).toEqual(new Set(tables.map((_, id) => id))) + await view.rollback() + view = undefined + const beforeRollback = await k('snapshot_journal_physical').orderBy(physicalKey.split(',')) + const rollbackHigh = (await k('snapshot_journal_clock').first()).revision + await k.transaction(async trx => { + await trx('tx_labels').where('txLabelId', 1).update({ label: 'rolled-back label' }) + await trx.rollback() + }) + expect(await k('snapshot_journal_physical').orderBy(physicalKey.split(','))).toEqual(beforeRollback) + expect((await k('snapshot_journal_clock').first()).revision).toBe(rollbackHigh) + for (const [tableId, table] of tables.entries()) { + const original = await k(table).first() + const identity = physicalIdentity(tableId, table, original) + const before = await k('snapshot_journal_physical').where(identity).first() + await replace(k, table, original) + await exact(k) + const after = await k('snapshot_journal_physical').where(identity).first() + expect(after.generation).toBeGreaterThan(before.generation) + } + expect(await k.raw('PRAGMA foreign_key_check')).toEqual([]) + } finally { + if (view) await view.rollback() + await reader.destroy() + await source.destroy() + await rm(directory, { recursive: true, force: true }) + } + }, + 60000 + ) + + async function journalFixture(): Promise { + const k = await fixture('BINARY', false, false) + const plan = await installGeneration(k) + for (let n = 0; n < 100; n++) if ((await copyGenerationPage(k, plan)).complete) break + await installCandidate(k, false) + return k + } + + test.each(tables)( + 'changing a %s key preserves the old tombstone and starts a new physical generation', + async table => { + const k = await journalFixture() + try { + const tableId = tables.indexOf(table) + await k(table).insert(value(table, 1, 1, 1)) + const before = await k('snapshot_journal_physical').where('tableId', tableId).first() + const original = keyOf(table, 1, 1) + const changed = keyOf(table, 2, 2) + await k(table).where(original).update(changed) + expect(await k(table).where(original)).toEqual([]) + expect(await k(table).where(changed)).toHaveLength(1) + const old = await k('snapshot_journal_physical') + .where({ + tableId, + id1: before.id1, + id2: before.id2, + exactText: before.exactText + }) + .first() + expect(old.present).toBe(0) + expect(old.generation).toBe(before.generation) + expect(old.revision).toBeGreaterThan(before.revision) + const current = await k('snapshot_journal_physical').where({ tableId, present: 1 }).first() + expect(current).toBeDefined() + expect(current.generation).toBeGreaterThan(before.generation) + expect(current.revision).toBe(current.generation) + } finally { + await k.destroy() + } + } + ) + + test.each(['sqlite3', 'better-sqlite3'])('SQLite observer alias %s prepares every table', async client => { + const k = await journalFixture() + k.client.config.client = client + try { + const definitions = await snapshotJournalSqliteObserverSql(k) + expect(definitions).toHaveLength(51) + } finally { + await k.destroy() + } + }) + + test('SQLite observer preparation refuses a foreign driver before querying its schema', async () => { + const raw = jest.fn(), + k = { client: { config: { client: 'mysql2' } }, raw } as unknown as Knex + await expect(snapshotJournalSqliteObserverSql(k)).rejects.toThrow('Snapshot journal observers require SQLite') + expect(raw).not.toHaveBeenCalled() + }) + + test('all thirteen observers preserve exact revision and generation values above 2^53', async () => { + const k = await journalFixture() + try { + await k('snapshot_journal_clock').update({ + ceiling: '9223372036854775807', + revision: '9007199254740992' + }) + for (const table of tables) await k(table).insert(value(table, 1, 1, 1)) + const rows = await k('snapshot_journal_physical') + .select('tableId', k.raw('CAST(revision AS TEXT) revision'), k.raw('CAST(generation AS TEXT) generation')) + .orderBy('tableId') + expect(rows.map(row => row.tableId)).toEqual(Array.from({ length: 13 }, (_, i) => i)) + for (const row of rows) { + expect(typeof row.revision).toBe('string') + expect(BigInt(row.revision)).toBeGreaterThan(9007199254740992n) + expect(row.generation).toBe(row.revision) + } + expect(await k('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 1, + reason: null + }) + } finally { + await k.destroy() + } + }) + + test.each(['capacity', 'signed63'])('%s exhaustion preserves all thirteen standard source writes', async kind => { + const k = await journalFixture() + try { + await k('snapshot_journal_clock').update( + kind === 'capacity' ? { ceiling: 1 } : { ceiling: '9223372036854775807', revision: '9223372036854775806' } + ) + for (const table of tables) await k(table).insert(value(table, 1, 1, 1)) + for (const table of tables) expect(await k(table).count('* AS n').first()).toEqual({ n: 1 }) + expect(await k('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 0, + reason: kind === 'capacity' ? 'capacity-exhausted' : 'revision-exhausted' + }) + const before = await k('snapshot_journal_physical').select('tableId', 'id1', 'id2', 'exactText') + await k('tx_labels').insert(value('tx_labels', 2, 2, 2)) + expect(await k('snapshot_journal_physical').select('tableId', 'id1', 'id2', 'exactText')).toEqual(before) + } finally { + await k.destroy() + } + }) + + test('400-byte field keys remain exact and 401 bytes invalidate without losing source data', async () => { + const k = await journalFixture() + try { + const accepted = '😀'.repeat(100), + oversized = accepted + 'x' + await k('certificates').insert(value('certificates', 1, 1, 1)) + await k('certificate_fields').insert({ + ...value('certificate_fields', 1, 1, 1), + fieldName: accepted + }) + expect(await k('snapshot_journal_clock').first('enabled')).toEqual({ + enabled: 1 + }) + expect((await k('snapshot_journal_scope').where({ tableId: 12, userId: 1 }).first()).exactText).toBe(accepted) + await k('certificate_fields').insert({ + ...value('certificate_fields', 1, 1, 1), + fieldName: oversized + }) + expect(await k('certificate_fields').pluck('fieldName')).toEqual([accepted, oversized]) + expect(await k('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 0, + reason: 'key-out-of-range' + }) + expect(await k('snapshot_journal_physical').where('exactText', oversized)).toEqual([]) + } finally { + await k.destroy() + } + }) + + test('an out-of-range numeric key invalidates atomically and rollback restores every observer', async () => { + const k = await journalFixture() + try { + const before = await k('snapshot_journal_clock').first() + await expect( + k.transaction(async t => { + await t.raw("INSERT INTO tx_labels(txLabelId,userId,label) VALUES(9007199254740992,1,'beyond exact wire ID')") + expect(await t('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 0, + reason: 'key-out-of-range' + }) + throw new Error('rollback invalidation') + }) + ).rejects.toThrow('rollback invalidation') + expect(await k('snapshot_journal_clock').first()).toEqual(before) + expect(await k('snapshot_journal_scope')).toEqual([]) + expect(await k('snapshot_journal_physical')).toEqual([]) + expect(await k('tx_labels')).toEqual([]) + await k.raw( + "INSERT INTO tx_labels(txLabelId,userId,label) VALUES(9007199254740992,1,'committed beyond exact wire ID')" + ) + expect(await k('tx_labels').count('* AS n').first()).toEqual({ n: 1 }) + expect(await k('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 0, + reason: 'key-out-of-range' + }) + } finally { + await k.destroy() + } + }) +}) + +// The receipt store's model is independent of SQL affected-row counts, database +// snapshots and the implementation's parser. It exercises actual SQLite commits. +test('bounded receipt schedules preserve exact committed identity, expiry, floor and capacity', async () => { + const Receipt = await import('./SnapshotJournalReceipt') + const ArchiveClock = await import('../archive/SnapshotArchiveSql') + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + let clock = 1000000 + const now = jest.spyOn(ArchiveClock, 'snapshotArchiveDatabaseNow').mockImplementation(async () => clock) + try { + await k.raw('PRAGMA busy_timeout=0') + for (const sql of Receipt.snapshotJournalReceiptDdl(k)) await k.raw(sql) + await k('snapshot_journal_retention').insert({ id: 1, floor: '0', receiptLimit: 3, receiptLifetimeMs: 1000 }) + await fc.assert( + fc.asyncProperty( + fc.array( + fc.record({ + kind: fc.integer({ min: 0, max: 4 }), + key: fc.integer({ min: 1, max: 8 }), + other: fc.boolean(), + rollback: fc.boolean(), + step: fc.integer({ min: 0, max: 80 }) + }), + { minLength: 1, maxLength: 32 } + ), + async commands => { + clock = 1000000 + let floor = 0n + const model = new Map< + string, + { requestId: string; binding: string; highWater: string; floor: string; expiresAt: number } + >() + await k('snapshot_journal_receipts').delete() + await k('snapshot_journal_retention').update({ floor: '0' }) + const binding = { + backend: '11'.repeat(32), + epoch: '12345678-1234-4234-9234-123456789012', + source: '22'.repeat(32), + schema: '33'.repeat(32), + storageIdentity: 'synthetic-receipt-model', + identityKey: '02' + '44'.repeat(32), + userId: 1, + chain: 'test' + } + const rollback = new Error('model rollback') + for (const command of commands) { + const high = 9007199254740992n + BigInt(command.key) + const request = { + requestId: command.key.toString(16).padStart(64, '0'), + highWater: snapshotJournalRevision(high.toString()), + expiresAt: 1000100 + command.key * 20 + } + const owner = { ...binding, userId: command.other ? 2 : 1 } + const digest = Receipt.snapshotJournalReceiptBinding(owner) + const prior = model.get(request.requestId) + async function recordCommand() { + const valid = + request.expiresAt > clock && + high >= floor && + (prior === undefined ? model.size < 3 : prior.binding === digest) + const result = { ...request, binding: digest, floor: floor.toString() } + const operation = k.transaction(async t => { + const value = await Receipt.recordSnapshotJournalReceipt(t, owner, request) + if (command.rollback) throw rollback + return value + }) + if (!valid || command.rollback) await expect(operation).rejects.toThrow() + else { + await expect(operation).resolves.toEqual(prior ?? result) + if (prior === undefined) model.set(request.requestId, result) + } + } + async function readCommand() { + const valid = + prior !== undefined && prior.binding === digest && request.expiresAt > clock && high >= floor + const operation = k.transaction(t => Receipt.readSnapshotJournalReceipt(t, owner, request)) + if (valid) await expect(operation).resolves.toEqual(prior) + else await expect(operation).rejects.toThrow() + } + async function collectCommand() { + const removed = [...model.values()].filter(row => row.expiresAt <= clock) + const operation = k.transaction(async t => { + expect(await Receipt.collectSnapshotJournalReceipts(t)).toBe(removed.length) + if (command.rollback) throw rollback + }) + if (command.rollback) await expect(operation).rejects.toThrow('model rollback') + else { + await operation + for (const row of removed) model.delete(row.requestId) + } + } + async function advanceTime() { + clock += command.step + } + async function advanceFloor() { + const next = high > floor ? high : floor + await k.transaction(async t => { + await t('snapshot_journal_retention').update({ floor: next.toString() }) + if (command.rollback) await t.rollback() + }) + if (!command.rollback) floor = next + } + if (command.kind === 0) await recordCommand() + else if (command.kind === 1) await readCommand() + else if (command.kind === 2) await collectCommand() + else if (command.kind === 3) await advanceTime() + else await advanceFloor() + const rows = await k('snapshot_journal_receipts').select('*').orderBy('requestId') + expect(rows).toEqual([...model.values()].sort((a, b) => a.requestId.localeCompare(b.requestId))) + expect(rows.length).toBeLessThanOrEqual(3) + } + } + ), + propertyParameters + ) + } finally { + now.mockRestore() + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalBootstrap.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalBootstrap.test.ts new file mode 100644 index 000000000..4038e9a63 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalBootstrap.test.ts @@ -0,0 +1,530 @@ +import { copySnapshotJournalBootstrapPage, SNAPSHOT_JOURNAL_BOOTSTRAP_DDL } from './SnapshotJournalBootstrap' +import { + snapshotJournalSqliteObserverSql, + SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL +} from './SnapshotJournalSqliteObservers' +import { installSnapshotJournalSqliteClock, SNAPSHOT_JOURNAL_SQLITE_WRITABLE } from './SnapshotJournalSqliteClock' +import { snapshotJournalRevision } from './SnapshotJournalRevision' +// Internal journal foundation. Registered migration and reader adoption remain separate. +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { seedArchiveClosure } from '../../../../test/utils/snapshotArchiveFixtures' +import { fixture, value, tables, numeric, profiles } from '../../../../test/utils/snapshotSqliteFixtures' +import { installGeneration } from '../../schema/snapshotSqliteIndexGeneration' +import { copyGenerationPage } from '../../schema/snapshotSqliteIndexBootstrap' + +const q = (name: string) => '"' + name.replaceAll('"', '""') + '"' +const scopeKey = 'tableId,userId,id1,id2,exactText' +const tuple = (table: string, prefix: string) => { + const key = numeric.find(([name]) => name === table)?.[1] + if (key) return [prefix + '.' + q(key), '0', "''"] + if (table === 'tx_labels_map') return [prefix + '.txLabelId', prefix + '.transactionId', "''"] + if (table === 'output_tags_map') return [prefix + '.outputTagId', prefix + '.outputId', "''"] + return [prefix + '.certificateId', '0', prefix + '.fieldName'] +} +async function installCandidate(k: Knex, reverse: boolean) { + const definitions = await snapshotJournalSqliteObserverSql(k) + await installSnapshotJournalSqliteClock(k, snapshotJournalRevision('1000000000')) + for (const ddl of SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL) await k.raw(ddl) + const prior: Array<{ name: string; sql: string }> = await k('sqlite_master') + .where('type', 'trigger') + .whereIn('tbl_name', tables) + .select('name', 'sql') + for (const definition of definitions) await k.raw(definition) + if (reverse) + for (const trigger of prior) { + await k.raw('DROP TRIGGER ??', [trigger.name]) + await k.raw(trigger.sql) + } +} +async function expected(k: Knex) { + const selections = profiles.map( + ([table, key], id) => `SELECT ${id} tableId,userId,${q(key)} id1,0 id2,'' exactText FROM ${q(table)}` + ) + for (const [id, table, left, leftKey, right, rightKey] of [ + [10, 'tx_labels_map', 'tx_labels', 'txLabelId', 'transactions', 'transactionId'], + [11, 'output_tags_map', 'output_tags', 'outputTagId', 'outputs', 'outputId'] + ]) + for (const [parent, key] of [ + [left, leftKey], + [right, rightKey] + ]) + selections.push( + `SELECT ${id},p.userId,m.${leftKey},m.${rightKey},'' FROM ${table} m JOIN ${parent} p ON m.${key}=p.${key}` + ) + selections.push( + 'SELECT 12,userId,certificateId,0,fieldName FROM certificate_fields', + 'SELECT 12,c.userId,f.certificateId,0,f.fieldName FROM certificate_fields f JOIN certificates c ON c.certificateId=f.certificateId', + "SELECT 9,t.userId,r.provenTxReqId,0,'' FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid", + "SELECT 8,t.userId,p.provenTxId,0,'' FROM transactions t JOIN proven_txs p ON p.provenTxId=t.provenTxId", + "SELECT 8,t.userId,p.provenTxId,0,'' FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid JOIN proven_txs p ON p.provenTxId=r.provenTxId" + ) + return await k.raw( + 'SELECT * FROM (' + selections.join(' UNION ') + ') ORDER BY tableId,userId,id1,id2,exactText COLLATE BINARY' + ) +} +async function exact(k: Knex) { + expect( + await k('snapshot_journal_scope').where('present', 1).select(scopeKey.split(',')).orderBy(scopeKey.split(',')) + ).toEqual(await expected(k)) + for (const [tableId, table] of tables.entries()) { + const keys = tuple(table, 's') + const rows = await k.raw( + `SELECT s.*,g.generation AS candidateGeneration,g.present AS candidatePresent FROM ${q(table)} s LEFT JOIN snapshot_journal_physical g ON g.tableId=${tableId} AND g.id1=${keys[0]} AND g.id2=${keys[1]} AND g.exactText COLLATE BINARY=${keys[2]} COLLATE BINARY` + ) + for (const row of rows) { + expect(row.candidatePresent).toBe(1) + expect(row.candidateGeneration).toBeGreaterThan(0) + } + } +} + +async function beginBootstrap(k: Knex) { + await installCandidate(k, false) + await k.raw(SNAPSHOT_JOURNAL_BOOTSTRAP_DDL) + await k('snapshot_journal_bootstrap').insert({ id: 1, stream: 0, cursor: null, rowLimit: 1000000, rowsUsed: 0 }) +} + +test('exact bootstrap copies bounded pages, resumes and preserves newer low-key observers', async () => { + const directory = await mkdtemp(join(tmpdir(), 'ts569-exact-bootstrap-')), + filename = join(directory, 'source.sqlite') + const k = knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + const writer = knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + try { + await k.raw('PRAGMA journal_mode=WAL') + await source.migrate('exact bootstrap fixture', 'synthetic-exact-bootstrap') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)), + { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, other.userId) + const original = await k('tx_labels').where('txLabelId', 1).first() + for (let start = 1000; start < 1600; start += 100) + await k('tx_labels').insert( + Array.from({ length: 100 }, (_, offset) => ({ + ...original, + txLabelId: start + offset, + label: 'bootstrap-' + (start + offset) + })) + ) + await beginBootstrap(k) + await k('snapshot_journal_clock').update({ + revision: '9007199254740992', + ceiling: '9223372036854775807' + }) + let moved = false, + complete = false, + max = 0, + insertQueries = 0 + const count = (q: { sql: string; bindings: unknown[] }) => { + if (/^insert into `snapshot_journal_(physical|scope)`/i.test(q.sql)) { + insertQueries++ + expect(q.bindings.length).toBeLessThanOrEqual(64 * 7) + } + } + k.on('query', count) + writer.on('query', count) + for (let page = 0; page < 100; page++) { + insertQueries = 0 + const result = await copySnapshotJournalBootstrapPage(page % 2 ? writer : k, 1000000) + expect(insertQueries).toBeLessThanOrEqual(4) + expect(result.invalidated).toBe(false) + max = Math.max(max, result.selected) + if (result.stream === 3 && result.selected === 256 && !moved) { + moved = true + await writer('tx_labels').where('txLabelId', 1).update({ label: 'changed below cursor' }) + await writer('tx_labels').where('txLabelId', 1000).delete() + await writer('tx_labels').insert({ + ...original, + txLabelId: 999, + label: 'inserted below cursor' + }) + await writer('tx_labels').where('txLabelId', 1599).update({ userId: other.userId }) + } + if (result.complete) { + complete = true + break + } + } + expect({ moved, complete, max }).toEqual({ moved: true, complete: true, max: 256 }) + await exact(k) + expect((await k('snapshot_journal_physical').where({ tableId: 3, id1: 1000 }).first()).present).toBe(0) + expect( + (await k('snapshot_journal_scope').where({ tableId: 3, id1: 1000, userId: user.userId }).first()).present + ).toBe(0) + const revisions = await k('snapshot_journal_physical').select( + k.raw('CAST(revision AS TEXT) revision'), + k.raw('CAST(generation AS TEXT) generation') + ) + for (const row of revisions) { + expect(BigInt(row.revision)).toBeGreaterThan(9007199254740992n) + expect(BigInt(row.generation)).toBeGreaterThan(9007199254740992n) + } + expect(await copySnapshotJournalBootstrapPage(k, 1000000)).toEqual({ + complete: true, + selected: 0, + stream: 17, + invalidated: false + }) + expect(await k.raw('PRAGMA foreign_key_check')).toEqual([]) + } finally { + await writer.destroy() + await source.destroy() + await rm(directory, { recursive: true, force: true }) + } +}, 60000) + +async function emptyFixture(): Promise { + const k = await fixture('BINARY', false, false) + const plan = await installGeneration(k) + for (let i = 0; i < 100; i++) if ((await copyGenerationPage(k, plan)).complete) break + return k +} + +test('oversized historical text is bounded before decode and disables bootstrap without losing source data', async () => { + const k = await emptyFixture() + try { + const text = 'x'.repeat(1000000) + await k('certificate_fields').insert({ + ...value('certificate_fields', 1, 1, 1), + fieldName: text + }) + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ stream: 12 }) + let maximum = 0 + const record = (response: unknown) => { + if (Array.isArray(response)) + for (const row of response) + if (row?.boundedText instanceof Uint8Array) maximum = Math.max(maximum, row.boundedText.length) + } + k.on('query-response', record) + const page = await copySnapshotJournalBootstrapPage(k, 1000000) + k.off('query-response', record) + expect(page).toEqual({ complete: false, selected: 1, stream: 12, invalidated: true }) + expect(maximum).toBe(401) + expect(await k('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 0, + reason: 'key-out-of-range' + }) + expect(await k('snapshot_journal_physical')).toEqual([]) + expect(await k('certificate_fields').select(k.raw('length(fieldName) n')).first()).toEqual({ + n: 1000000 + }) + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 12, + cursor: null, + rowLimit: 1000000, + rowsUsed: 0 + }) + } finally { + await k.destroy() + } +}) + +test.each(['capacity', 'signed63'])('%s exhaustion commits invalidation without publishing progress', async kind => { + const k = await emptyFixture() + try { + await k('transactions').insert(value('transactions', 1, 1, 1)) + await beginBootstrap(k) + await k('snapshot_journal_clock').update( + kind === 'capacity' + ? { revision: 1, ceiling: 1 } + : { revision: '9223372036854775807', ceiling: '9223372036854775807' } + ) + expect(await copySnapshotJournalBootstrapPage(k, 1000000)).toEqual({ + complete: false, + selected: 1, + stream: 0, + invalidated: true + }) + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: null, + rowLimit: 1000000, + rowsUsed: 0 + }) + expect(await k('snapshot_journal_physical')).toEqual([]) + expect(await k('transactions').count('* AS n').first()).toEqual({ n: 1 }) + } finally { + await k.destroy() + } +}) + +test.each(['{', '{}', '[]', '["wrong-type"]', '[9007199254740992]', '[1,2]'])( + 'malformed persisted cursor %s refuses before progress', + async cursor => { + const k = await emptyFixture() + try { + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ cursor }) + await expect(copySnapshotJournalBootstrapPage(k, 1000000)).rejects.toThrow('bootstrap state') + expect(await k('snapshot_journal_physical')).toEqual([]) + } finally { + await k.destroy() + } + } +) + +test('historical BLOB field spelling invalidates instead of changing its SQL comparison domain', async () => { + const k = await emptyFixture() + try { + await k.raw( + "INSERT INTO certificate_fields(userId,certificateId,fieldName,fieldValue) VALUES(1,1,CAST('field' AS BLOB),'value')" + ) + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ stream: 12 }) + expect((await copySnapshotJournalBootstrapPage(k, 1000000)).invalidated).toBe(true) + expect(await k('snapshot_journal_physical')).toEqual([]) + expect(await k('certificate_fields').select(k.raw('typeof(fieldName) kind')).first()).toEqual({ + kind: 'blob' + }) + } finally { + await k.destroy() + } +}) + +test('malformed SQLite clock response rolls back allocation and bootstrap progress', async () => { + const k = await emptyFixture() + let altered = false + const listener = (row: { enabled: number }, query: { sql: string }) => { + if (query.sql.startsWith('select `enabled`') && query.sql.includes('as `value`')) { + row.enabled = 2 + altered = true + } + } + try { + await k('transactions').insert(value('transactions', 1, 1, 1)) + await beginBootstrap(k) + const before = await k('snapshot_journal_clock').first() + k.on('query-response', listener) + await expect(copySnapshotJournalBootstrapPage(k, 1000000)).rejects.toThrow('Invalid snapshot') + expect(altered).toBe(true) + expect(await k('snapshot_journal_clock').first()).toEqual(before) + expect(await k('snapshot_journal_physical')).toEqual([]) + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: null, + rowLimit: 1000000, + rowsUsed: 0 + }) + } finally { + k.off('query-response', listener) + await k.destroy() + } +}) + +test('legacy SQLite alias bootstraps an exact 400-byte historical key', async () => { + const k = await emptyFixture() + k.client.config.client = 'sqlite3' + try { + const fieldName = '😀'.repeat(100) + await k('certificate_fields').insert({ ...value('certificate_fields', 1, 1, 1), fieldName }) + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ stream: 12 }) + expect(await copySnapshotJournalBootstrapPage(k, 1000000)).toMatchObject({ + selected: 1, + invalidated: false + }) + expect((await k('snapshot_journal_physical').first()).exactText).toBe(fieldName) + expect(await copySnapshotJournalBootstrapPage(k, 1000000)).toMatchObject({ + selected: 0, + invalidated: false + }) + } finally { + await k.destroy() + } +}) + +test('SQLite bootstrap commits over-budget invalidation without source loss or partial metadata', async () => { + const k = await emptyFixture() + try { + await k('transactions').insert([value('transactions', 1, 1, 1), value('transactions', 2, 2, 1)]) + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ rowLimit: null }) + const clock = await k('snapshot_journal_clock').first() + expect(await copySnapshotJournalBootstrapPage(k, 1)).toEqual({ + complete: false, + selected: 2, + stream: 0, + invalidated: true + }) + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: null, + rowLimit: 1, + rowsUsed: 0 + }) + expect(await k('snapshot_journal_physical')).toEqual([]) + expect(await k('snapshot_journal_scope')).toEqual([]) + expect(await k('snapshot_journal_clock').first()).toEqual({ ...clock, enabled: 0, reason: 'capacity-exhausted' }) + await k('transactions').insert(value('transactions', 3, 3, 1)) + expect(await k('transactions')).toHaveLength(3) + expect(await k('snapshot_journal_physical')).toEqual([]) + await expect(copySnapshotJournalBootstrapPage(k, 3)).rejects.toThrow('Invalid snapshot') + } finally { + await k.destroy() + } +}) + +test('SQLite bootstrap commits the exact allowance with its cursor and preserves it on resume', async () => { + const k = await emptyFixture() + try { + await k('transactions').insert([value('transactions', 1, 1, 1), value('transactions', 2, 2, 1)]) + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ rowLimit: null }) + expect((await copySnapshotJournalBootstrapPage(k, 2)).invalidated).toBe(false) + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: '[2]', + rowLimit: 2, + rowsUsed: 2 + }) + expect(await k('snapshot_journal_physical')).toHaveLength(2) + expect((await copySnapshotJournalBootstrapPage(k, 2)).selected).toBe(0) + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 1, + cursor: null, + rowLimit: 2, + rowsUsed: 2 + }) + await expect(copySnapshotJournalBootstrapPage(k, 1)).rejects.toThrow('Invalid snapshot') + } finally { + await k.destroy() + } +}) + +test('SQLite bootstrap rolls back a newly bound allowance, charge, cursor and revision on failed metadata', async () => { + const k = await emptyFixture() + try { + await k('transactions').insert(value('transactions', 1, 1, 1)) + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ rowLimit: null }) + const clock = await k('snapshot_journal_clock').first() + await k.raw( + "CREATE TRIGGER fail_bootstrap_metadata BEFORE INSERT ON snapshot_journal_physical BEGIN SELECT RAISE(ABORT,'fixture metadata failure'); END" + ) + await expect(copySnapshotJournalBootstrapPage(k, 1)).rejects.toThrow('fixture metadata failure') + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: null, + rowLimit: null, + rowsUsed: 0 + }) + expect(await k('snapshot_journal_clock').first()).toEqual(clock) + expect(await k('snapshot_journal_physical')).toEqual([]) + await k.raw('DROP TRIGGER fail_bootstrap_metadata') + await copySnapshotJournalBootstrapPage(k, 1) + expect((await k('snapshot_journal_bootstrap').first()).rowsUsed).toBe(1) + } finally { + await k.destroy() + } +}) + +test.each([0, 2147483647])('SQLite bootstrap accepts empty-stream budget boundary %s', async allowance => { + const k = await emptyFixture() + try { + await beginBootstrap(k) + await k('snapshot_journal_bootstrap').update({ rowLimit: null }) + expect((await copySnapshotJournalBootstrapPage(k, allowance)).selected).toBe(0) + expect(await k('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 1, + cursor: null, + rowLimit: allowance, + rowsUsed: 0 + }) + } finally { + await k.destroy() + } +}) + +test.each(['BINARY', 'NOCASE', 'RTRIM'])( + 'SQLite journal source probes use complete point indexes under %s without widening byte identity', + async collation => { + const k = await fixture(collation, false, false) + try { + for (const table of tables) await k(table).insert(value(table, 1, 2, 1)) + const definitions = await snapshotJournalSqliteObserverSql(k) + for (const [tableId, table] of tables.entries()) { + const definition = definitions.find(sql => + sql.startsWith('CREATE TRIGGER snapshot_journal_physical_' + tableId + '_INSERT ') + )! + const start = definition.indexOf('EXISTS(') + 'EXISTS('.length + const end = definition.indexOf(') WHERE ' + SNAPSHOT_JOURNAL_SQLITE_WRITABLE, start) + expect(start).toBeGreaterThan('EXISTS('.length) + expect(end).toBeGreaterThan(start) + const row = value(table, 1, 2, 1) + const names: string[] = [] + const query = definition + .slice(start, end) + .replace( + /NEW\.(?:"([^"]+)"|([A-Za-z][A-Za-z0-9_]*))/g, + (_match, quoted: string | undefined, bare: string | undefined) => { + const name = (quoted ?? bare)! + names.push(name) + // Driver numeric bindings are doubles; NEW/OLD key columns are native integers. + return typeof row[name] === 'number' ? 'CAST(? AS INTEGER)' : '?' + } + ) + const bindings = names.map(name => row[name]) as Knex.RawBinding[] + const plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + query, bindings) + expect(plan.some(step => step.detail.startsWith('SCAN s'))).toBe(false) + const search = plan.find(step => step.detail.startsWith('SEARCH s ')) + expect(search).toBeDefined() + expect(search!.detail.match(/=\?/g)).toHaveLength(tableId < 10 ? 1 : 2) + expect(await k.raw(query, bindings)).toHaveLength(1) + if (table === 'certificate_fields') { + for (const alternate of [String(row.fieldName).toUpperCase(), String(row.fieldName) + ' ']) { + const changed = names.map(name => (name === 'fieldName' ? alternate : row[name])) as Knex.RawBinding[] + expect(await k.raw(query, changed)).toHaveLength(0) + } + } + } + } finally { + await k.destroy() + } + } +) + +test('SQLite bootstrap rejects caller transactions before any journal access', async () => { + const k = await emptyFixture() + try { + await beginBootstrap(k) + const before = await k('snapshot_journal_bootstrap').first() + await k.transaction(async t => { + const queries: string[] = [] + const observe = (query: { sql: string }) => queries.push(query.sql) + t.on('query', observe) + try { + await expect(copySnapshotJournalBootstrapPage(t, 1000000)).rejects.toThrow('Invalid snapshot') + expect(queries).toEqual([]) + } finally { + t.off('query', observe) + } + }) + expect(await k('snapshot_journal_bootstrap').first()).toEqual(before) + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalBootstrap.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalBootstrap.ts new file mode 100644 index 000000000..14dee4808 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalBootstrap.ts @@ -0,0 +1,419 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { numeric, legacyNames } from '../../schema/snapshotSqliteMembership' +import { names } from '../../schema/snapshotSqliteIndexGeneration' +import { SNAPSHOT_JOURNAL_SQLITE_ADVANCE } from './SnapshotJournalSqliteClock' +import { + compareSnapshotJournalRevisions, + MAX_SNAPSHOT_JOURNAL_REVISION, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' +import { snapshotJournalRevisionText, snapshotJournalRevisionOperand } from './SnapshotJournalRevisionSql' + +export const SNAPSHOT_JOURNAL_BOOTSTRAP_DDL = + 'CREATE TABLE snapshot_journal_bootstrap(id INTEGER NOT NULL PRIMARY KEY,stream INTEGER NOT NULL,`cursor` TEXT,rowLimit INTEGER,rowsUsed INTEGER NOT NULL,CHECK(id=1),CHECK(stream BETWEEN 0 AND 17),CHECK(rowLimit IS NULL OR rowLimit BETWEEN 0 AND 2147483647),CHECK(rowsUsed BETWEEN 0 AND 2147483647),CHECK(rowLimit IS NULL OR rowsUsed<=rowLimit))' +interface Stream { + table: string + keys: string[] + text?: string + extra?: string + physical: boolean + record: (row: Record) => Record +} +/** A conservative charge counts every examined bootstrap record, including an + * already-observed key. The bound survives retries without counting SQL affected + * rows, whose semantics differ between the two drivers. */ +export function validSnapshotJournalBootstrapBudget(row: { rowLimit: unknown; rowsUsed: unknown }): boolean { + const bounded = (value: unknown): value is number => + typeof value === 'number' && Number.isInteger(value) && value >= 0 && value <= 2147483647 + return ( + bounded(row.rowsUsed) && + (row.rowLimit === null ? row.rowsUsed === 0 : bounded(row.rowLimit) && row.rowsUsed <= row.rowLimit) + ) +} +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid snapshot journal bootstrap state or source order') +} +function sqlite(k: Knex): boolean { + const client = k.client.config.client + if (client === 'better-sqlite3' || client === 'sqlite3') return true + if (client === 'mysql' || client === 'mysql2') return false + return invalid() +} +function physicalKeys(table: string, key: string | undefined): string[] { + if (key) return [key] + if (table === 'tx_labels_map') return ['txLabelId', 'transactionId'] + if (table === 'output_tags_map') return ['outputTagId', 'outputId'] + return ['fieldName', 'certificateId'] +} +function physicalRecord( + tableId: number, + key: string | undefined, + row: Record +): Record { + let id1 = row.certificateId, + id2: unknown = 0 + if (key) id1 = row[key] + else if (tableId === 10) { + id1 = row.txLabelId + id2 = row.transactionId + } else if (tableId === 11) { + id1 = row.outputTagId + id2 = row.outputId + } + return { tableId, id1, id2, exactText: tableId === 12 ? row.fieldName : '', present: 1 } +} +function streams(local: boolean): Stream[] { + const membership = local ? names : legacyNames + const tables = [...numeric.map(source => source.table), 'tx_labels_map', 'output_tags_map', 'certificate_fields'] + const physical: Stream[] = tables.map((table, tableId) => { + const key = numeric[tableId]?.key + return { + table, + keys: physicalKeys(table, key), + text: tableId === 12 ? 'fieldName' : undefined, + physical: true, + record: row => physicalRecord(tableId, key, row) + } + }) + return [ + ...physical, + { + table: membership.profile, + keys: ['snapshotTableId', 'snapshotUserId', 'snapshotRowId'], + physical: false, + record: r => ({ + tableId: r.snapshotTableId, + userId: r.snapshotUserId, + id1: r.snapshotRowId, + id2: 0, + exactText: '', + present: 1 + }) + }, + { + table: membership.relation, + keys: ['snapshotTableId', 'snapshotUserId', 'snapshotLeftId', 'snapshotRightId'], + extra: 'snapshotMembership', + physical: false, + record: r => ({ + tableId: Number(r.snapshotTableId) + 10, + userId: r.snapshotUserId, + id1: r.snapshotLeftId, + id2: r.snapshotRightId, + exactText: '', + present: Number(r.snapshotMembership) !== 0 ? 1 : 0 + }) + }, + { + table: membership.certificate, + keys: ['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId'], + text: 'snapshotFieldName', + extra: 'snapshotMembership', + physical: false, + record: r => ({ + tableId: 12, + userId: r.snapshotUserId, + id1: r.snapshotCertificateId, + id2: 0, + exactText: r.snapshotFieldName, + present: Number(r.snapshotMembership) !== 0 ? 1 : 0 + }) + }, + { + table: membership.keys, + keys: ['tableId', 'userId', 'rowId'], + extra: 'present', + physical: false, + record: r => ({ + tableId: r.tableId === 0 ? 9 : 8, + userId: r.userId, + id1: r.rowId, + id2: 0, + exactText: '', + present: Number(r.present) + }) + } + ] +} +function validKey(record: Record): boolean { + const integer = (value: unknown, min: number, max = Number.MAX_SAFE_INTEGER) => + typeof value === 'number' && Number.isSafeInteger(value) && value >= min && value <= max + const text = record.exactText + return ( + integer(record.tableId, 0, 12) && + integer(record.id1, 1) && + integer(record.id2, 0) && + (record.userId === undefined || integer(record.userId, 1)) && + typeof text === 'string' && + Buffer.byteLength(text, 'utf8') <= 400 && + Buffer.from(text, 'utf8').toString('utf8') === text && + (record.present === 0 || record.present === 1) + ) +} +async function invalidate(k: Knex, reason: string): Promise { + if (sqlite(k)) await k('snapshot_journal_clock').where({ id: 1, enabled: 1 }).update({ enabled: 0, reason }) + else await k('snapshot_journal_invalid').insert({ id: 1, reason }).onConflict('id').ignore() +} +async function revision(k: Knex, local: boolean): Promise { + if (local) { + await k.raw(SNAPSHOT_JOURNAL_SQLITE_ADVANCE) + const row = await k('snapshot_journal_clock') + .where('id', 1) + .select('enabled', { value: snapshotJournalRevisionText(k, 'revision') }) + .first() + if (!row || ![0, 1].includes(row.enabled)) return invalid() + return row.enabled === 1 ? snapshotJournalRevision(row.value) : undefined + } + const row = await k('snapshot_journal_clock') + .where('id', 1) + .select({ ceiling: snapshotJournalRevisionText(k, 'ceiling') }) + .first() + if (!row) return invalid() + const ceiling = snapshotJournalRevision(row.ceiling) + await k('snapshot_journal_events').insert({}) + const [[allocated]]: Array> = await k.raw('SELECT CAST(LAST_INSERT_ID() AS CHAR) value') + if (!allocated || typeof allocated.value !== 'string' || !/^\d+$/.test(allocated.value)) return invalid() + await k('snapshot_journal_events').where('revision', allocated.value).delete() + if (BigInt(allocated.value) > BigInt(MAX_SNAPSHOT_JOURNAL_REVISION)) { + await invalidate(k, 'revision-exhausted') + return undefined + } + const value = snapshotJournalRevision(allocated.value) + if (compareSnapshotJournalRevisions(value, ceiling) > 0) { + await invalidate(k, 'capacity-exhausted') + return undefined + } + return value +} +async function mysqlSourceQuery(k: Knex, stream: Stream): Promise<{ query: Knex.QueryBuilder }> { + const [parts]: Array> = await k.raw( + 'SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX', + [stream.table] + ) + const indexes = new Map() + for (const part of parts) { + if (!indexes.has(part.name)) indexes.set(part.name, []) + indexes.get(part.name)!.push(part.field) + } + const index = [...indexes].find(([, fields]) => JSON.stringify(fields) === JSON.stringify(stream.keys))?.[0] + if (!index) return invalid() + return { query: k.from(k.raw('?? AS ?? FORCE INDEX (??)', [stream.table, 's', index])).forShare() } +} +function decodeTextRows(rows: Array>, stream: Stream, local: boolean): void { + if (!stream.text) return + for (const row of rows) { + const bytes = row.boundedText + if (!(bytes instanceof Uint8Array)) return invalid() + const text = Buffer.from(bytes).toString('utf8') + row[stream.text] = + (!local || row.boundedTextType === 'text') && Buffer.from(text, 'utf8').equals(Buffer.from(bytes)) + ? text + : undefined + delete row.boundedTextType + delete row.boundedText + } +} +function projectPage(k: Knex, query: Knex.QueryBuilder, stream: Stream, local: boolean): void { + for (const key of stream.keys) { + if (key === stream.text) { + if (local) query.select(k.raw('typeof(??) AS ??', ['s.' + key, 'boundedTextType'])) + query.select( + k.raw(local ? 'substr(CAST(?? AS BLOB),1,401) AS ??' : 'SUBSTRING(CAST(?? AS BINARY),1,401) AS ??', [ + 's.' + key, + 'boundedText' + ]) + ) + } else query.select('s.' + key) + } + if (stream.extra) query.select('s.' + stream.extra) +} +function afterSqliteCursor(query: Knex.QueryBuilder, stream: Stream, cursor: Array): void { + query.whereRaw('(' + stream.keys.map(() => '??').join(',') + ') > (' + stream.keys.map(() => '?').join(',') + ')', [ + ...stream.keys.map(key => 's.' + key), + ...cursor + ]) +} +function afterMysqlCursor(query: Knex.QueryBuilder, stream: Stream, cursor: Array): void { + query.where(function () { + stream.keys.forEach((key, index) => { + this.orWhere(function () { + for (let i = 0; i < index; i++) this.where('s.' + stream.keys[i], cursor[i]) + this.where('s.' + key, '>', cursor[index]) + }) + }) + }) +} +async function queryPage( + k: Knex, + stream: Stream, + cursor: Array | undefined, + local: boolean +): Promise>> { + const query = local ? k.from({ s: stream.table }) : (await mysqlSourceQuery(k, stream)).query + projectPage(k, query, stream, local) + query.orderBy(stream.keys.map(key => 's.' + key)).limit(256) + if (cursor) { + if (local) afterSqliteCursor(query, stream, cursor) + else afterMysqlCursor(query, stream, cursor) + } + const sql = query.toSQL() + const plan = await k.raw((local ? 'EXPLAIN QUERY PLAN ' : 'EXPLAIN ') + sql.sql, sql.bindings as Knex.RawBinding[]) + if ( + (local ? plan : plan[0]).some((step: { detail?: string; Extra?: string }) => + local ? step.detail?.includes('TEMP B-TREE') : step.Extra?.includes('filesort') + ) + ) + return invalid() + const rows: Array> = await query + if (rows.length > 256) return invalid() + decodeTextRows(rows, stream, local) + return rows +} + +function bootstrapCursor(stream: Stream, value: string | null): Array | undefined { + if (value === null) return undefined + if (Buffer.byteLength(value) > 2048) return invalid() + let parsed: unknown + try { + parsed = JSON.parse(value) + } catch { + return invalid() + } + if (!Array.isArray(parsed) || parsed.length !== stream.keys.length) return invalid() + for (const [i, value] of parsed.entries()) + if ( + stream.keys[i] === stream.text + ? typeof value !== 'string' || Buffer.byteLength(value) > 400 + : typeof value !== 'number' || !Number.isSafeInteger(value) || value < 0 + ) + return invalid() + return parsed as Array +} +interface BootstrapState { + stream: number + cursor: string | null + rowLimit: number | null + rowsUsed: number +} +async function readBootstrapState(t: Knex, streamCount: number, rowLimit: number): Promise { + const state: BootstrapState | undefined = await t('snapshot_journal_bootstrap').where('id', 1).first() + if ( + !state || + !Number.isInteger(state.stream) || + state.stream < 0 || + state.stream > streamCount || + !(state.cursor === null || typeof state.cursor === 'string') || + !validSnapshotJournalBootstrapBudget(state) || + (state.rowLimit !== null && state.rowLimit !== rowLimit) || + (state.rowLimit === null && (state.stream !== 0 || state.cursor !== null)) + ) + return invalid() + return state +} +async function writeBootstrapRecords( + t: Knex, + stream: Stream, + records: Array>, + local: boolean +): Promise { + if (!records.length) return true + const at = await revision(t, local) + if (at === undefined) return false + const table = stream.physical ? 'snapshot_journal_physical' : 'snapshot_journal_scope' + const primary = stream.physical + ? ['tableId', 'id1', 'id2', 'exactText'] + : ['tableId', 'userId', 'id1', 'id2', 'exactText'] + // At most four writes per page, each below the 999-binding SQLite floor. + const batches = Array.from({ length: Math.ceil(records.length / 64) }, (_, i) => records.slice(i * 64, (i + 1) * 64)) + await runInSeries(batches, async batch => { + const query = t(table) + .insert( + batch.map(record => ({ + ...record, + exactText: local ? record.exactText : Buffer.from(record.exactText as string), + revision: snapshotJournalRevisionOperand(t, at), + ...(stream.physical ? { generation: snapshotJournalRevisionOperand(t, at) } : {}) + })) + ) + .onConflict(primary) + if (local) await query.ignore() + else await query.merge({ id1: t.ref(table + '.id1') }) + }) + return true +} +export interface SnapshotJournalBootstrapPage { + complete: boolean + selected: number + stream: number + invalidated: boolean +} +/** Fresh/resumed owned state only. Caller validates migration ownership before + * every resume and supplies an explicit row allowance. The first page persists + * that allowance; changing it requires a new generation. This bounds bootstrap + * allocation, separately from event-window and physical/receipt retention limits. + * Owns its transaction: a caller transaction could retain the writer barrier + * beyond this page or reuse an older MySQL progress/capacity read view. + */ +export async function copySnapshotJournalBootstrapPage( + k: Knex, + rowLimit: number +): Promise { + if ( + k.isTransaction || + !Number.isSafeInteger(rowLimit) || + !validSnapshotJournalBootstrapBudget({ rowLimit, rowsUsed: 0 }) + ) + return invalid() + const local = sqlite(k), + all = streams(local) + return await k.transaction(async t => { + if (local) + await t('snapshot_journal_bootstrap') + .where('id', 1) + .update({ stream: t.ref('stream') }) + else if (!(await t('snapshot_journal_clock').where('id', 1).forUpdate().noWait().first('id'))) return invalid() + const state = await readBootstrapState(t, all.length, rowLimit) + const enabled = local + ? (await t('snapshot_journal_clock').where('id', 1).first('enabled'))?.enabled === 1 + : !(await t('snapshot_journal_invalid').where('id', 1).first('id')) + if (!enabled) return { complete: false, selected: 0, stream: state.stream, invalidated: true } + if (state.rowLimit === null) { + await t('snapshot_journal_bootstrap').where('id', 1).update({ rowLimit }) + } + if (state.stream === all.length) { + if (state.cursor !== null) return invalid() + return { complete: true, selected: 0, stream: state.stream, invalidated: false } + } + const stream = all[state.stream] + const cursor = bootstrapCursor(stream, state.cursor) + const rows = await queryPage(t, stream, cursor, local), + records = rows.map(row => stream.record(row)) + if (records.some(record => !validKey(record))) { + await invalidate(t, 'key-out-of-range') + return { complete: false, selected: rows.length, stream: state.stream, invalidated: true } + } + if (records.length > rowLimit - state.rowsUsed) { + await invalidate(t, 'capacity-exhausted') + return { complete: false, selected: rows.length, stream: state.stream, invalidated: true } + } + if (!(await writeBootstrapRecords(t, stream, records, local))) + return { complete: false, selected: rows.length, stream: state.stream, invalidated: true } + const last = rows.at(-1) + await t('snapshot_journal_bootstrap') + .where('id', 1) + .update( + last + ? { cursor: JSON.stringify(stream.keys.map(key => last[key])), rowsUsed: state.rowsUsed + rows.length } + : { stream: state.stream + 1, cursor: null } + ) + return { + complete: !last && state.stream + 1 === all.length, + selected: rows.length, + stream: state.stream, + invalidated: false + } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCapture.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCapture.test.ts new file mode 100644 index 000000000..e6872310f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCapture.test.ts @@ -0,0 +1,866 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { seedArchiveClosure } from '../../../../test/utils/snapshotArchiveFixtures' +import { + installSnapshotJournalSqliteGeneration, + completeSnapshotJournalSqliteGeneration +} from './SnapshotJournalSqliteGeneration' +import { copySnapshotJournalBootstrapPage } from './SnapshotJournalBootstrap' +import { readSnapshotJournalReceipt } from './SnapshotJournalReceipt' +import { snapshotJournalRevision } from './SnapshotJournalRevision' +import { SnapshotJournalConnectionCleanupError } from './SnapshotJournalConnections' +import * as Connections from './SnapshotJournalConnections' +import type { SnapshotJournalCaptureRequest, SnapshotJournalSource } from './SnapshotJournalCapture' +import * as Capture from './SnapshotJournalCapture' +import * as Closure from '../archive/KnexSnapshotArchiveClosure' +import * as Backend from './SnapshotJournalCaptureBackend' +import * as Receipts from './SnapshotJournalReceipt' +import { snapshotArchiveTables } from '../archive/KnexSnapshotArchiveStore' +import { WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import * as Archive from '../archive/KnexSnapshotArchiveSource' +import * as MysqlGeneration from './SnapshotJournalMysqlGeneration' +import * as Fence from './SnapshotJournalCaptureFence' +import * as ArchiveSql from '../archive/SnapshotArchiveSql' +import { createHash } from 'node:crypto' +import { Duplex } from 'node:stream' + +const identity = '02' + '11'.repeat(32) +let request: SnapshotJournalCaptureRequest +beforeEach(() => { + request = { + ceiling: snapshotJournalRevision('1000000'), + receiptPolicy: { receiptLimit: 128, receiptLifetimeMs: 600000 } + } +}) +function gate() { + let resolve!: () => void + const promise = new Promise(yes => { + resolve = yes + }) + return { promise, resolve } +} +async function fixture(complete = true) { + const directory = await mkdtemp(join(tmpdir(), 'ts569-capture-controller-')) + const k = knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + try { + await storage.migrate('capture controller', 'synthetic-capture-controller') + await storage.makeAvailable() + await k.raw('PRAGMA journal_mode = WAL') + const { user } = await storage.findOrInsertUser(identity) + const { user: foreign } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(storage, user.userId, foreign.userId) + await installSnapshotJournalSqliteGeneration(k, request.ceiling, request.receiptPolicy) + if (complete) { + let finished = false + for (let n = 0; n < 80 && !finished; n++) finished = (await copySnapshotJournalBootstrapPage(k, 1000000)).complete + expect(finished).toBe(true) + await completeSnapshotJournalSqliteGeneration(k, request.receiptPolicy) + } + return { + k, + storage, + directory, + close: async () => { + await storage.destroy() + await rm(directory, { recursive: true, force: true }) + } + } + } catch (error) { + await storage.destroy() + await rm(directory, { recursive: true, force: true }) + throw error + } +} + +afterEach(() => jest.restoreAllMocks()) + +test('all thirteen tables retain their pinned values while independent writers modify every table', async () => { + const f = await fixture() + const view = await f.storage.openSnapshotJournalSource(identity, request) + const raw = [ + 'proven_txs', + 'proven_tx_reqs', + 'output_baskets', + 'transactions', + 'commissions', + 'outputs', + 'output_tags', + 'output_tags_map', + 'tx_labels', + 'tx_labels_map', + 'certificates', + 'certificate_fields', + 'sync_states' + ] + try { + expect(snapshotArchiveTables).toHaveLength(13) + const before = [] + for (const table of snapshotArchiveTables) { + const page = await view.readPage(table) + expect(page.done).toBe(true) + expect(page.rows.length).toBeGreaterThan(0) + before.push(page.rows) + } + for (const table of raw) await f.k(table).update({ created_at: new Date('2026-01-02T00:00:00Z').getTime() }) + for (const [index, table] of snapshotArchiveTables.entries()) { + const page = await view.readPage(table) + expect(page.rows).toEqual(before[index]) + for (const row of page.rows) if ('userId' in row) expect(row.userId).toBe(view.user.userId) + } + expect(before[0].map(row => ('provenTxId' in row ? row.provenTxId : undefined))).toEqual([1, 3]) + expect(before[1].map(row => ('provenTxReqId' in row ? row.provenTxReqId : undefined))).toEqual([1, 3]) + expect(before[12].map(row => ('syncStateId' in row ? row.syncStateId : undefined))).toEqual([1, 3]) + } finally { + await view.close() + await f.close() + } +}) + +test('profile closure verification holds no writer barrier and publishes only after verification finishes', async () => { + const f = await fixture(), + entered = gate(), + release = gate() + const verify = Closure.assertKnexSnapshotArchiveClosure + jest.spyOn(Closure, 'assertKnexSnapshotArchiveClosure').mockImplementation(async (...args) => { + entered.resolve() + await release.promise + await verify(...args) + }) + let view: SnapshotJournalSource | undefined + try { + const opening = f.storage.openSnapshotJournalSource(identity, request) + let published = false + void opening.then( + () => { + published = true + }, + () => undefined + ) + await entered.promise + expect(await f.k('snapshot_journal_receipts')).toHaveLength(1) + await f.k('tx_labels').where('txLabelId', 1).update({ label: 'foreground during closure' }) + expect(published).toBe(false) + release.resolve() + view = await opening + expect((await view.readPage('txLabels')).rows.some(row => row.label === 'foreground during closure')).toBe(false) + } finally { + release.resolve() + await view?.close() + await f.close() + } +}) + +test.each(['receipt before commit', 'closure after commit'] as const)( + 'cancellation during %s rejects promptly and drains before source capacity can reopen', + async phase => { + const f = await fixture(), + entered = gate(), + release = gate(), + controller = new AbortController() + if (phase === 'receipt before commit') { + const record = Receipts.recordSnapshotJournalReceipt + jest.spyOn(Receipts, 'recordSnapshotJournalReceipt').mockImplementation(async (...args) => { + const result = await record(...args) + entered.resolve() + await release.promise + return result + }) + } else { + const verify = Closure.assertKnexSnapshotArchiveClosure + jest.spyOn(Closure, 'assertKnexSnapshotArchiveClosure').mockImplementation(async (...args) => { + entered.resolve() + await release.promise + await verify(...args) + }) + } + try { + const opening = f.storage.openSnapshotJournalSource(identity, request, { + signal: controller.signal + }) + const rejection = expect(opening).rejects.toThrow('cancelled') + await entered.promise + controller.abort() + await rejection + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('already has') + let drained = false + const cleanup = f.storage.awaitSnapshotJournalCaptureCleanup().then(() => { + drained = true + }) + await Promise.resolve() + expect(drained).toBe(false) + release.resolve() + await cleanup + expect(await f.k('snapshot_journal_receipts')).toHaveLength(phase === 'closure after commit' ? 1 : 0) + jest.restoreAllMocks() + const next = await f.storage.openSnapshotJournalSource(identity, request) + expect(await f.k('snapshot_journal_receipts')).toHaveLength(phase === 'closure after commit' ? 2 : 1) + await next.close() + } finally { + release.resolve() + jest.restoreAllMocks() + await f.close() + } + } +) + +test('provider publishes only a committed receipt, serves immutable profile pages and releases its single source admission', async () => { + const f = await fixture() + let view: SnapshotJournalSource | undefined + try { + const labels = await f.storage.findTxLabels({ partial: { userId: 1 } }) + view = await f.storage.openSnapshotJournalSource(identity, request) + expect(view.sourceStorage.chain).toBe('test') + expect(view.user.identityKey).toBe(identity) + expect(await f.k.transaction(t => readSnapshotJournalReceipt(t, view!.receiptBinding, view!.receipt))).toEqual( + view.receipt + ) + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('already has') + await expect(f.storage.openSnapshotArchiveSource(identity)).rejects.toThrow('already has') + await f.k('tx_labels').where('txLabelId', 1).update({ label: 'after capture' }) + const page = await view.readPage('txLabels') + expect(page.rows.map(row => row.label)).toEqual(labels.map(row => row.label)) + expect(page.rows.every(row => row.userId === view!.user.userId)).toBe(true) + const rows: Array<{ revision: string }> = await f + .k('snapshot_journal_physical') + .select(f.k.raw('CAST(revision AS TEXT) revision')) + expect(rows.some(row => BigInt(row.revision) > BigInt(view!.receipt.highWater))).toBe(true) + await view.close() + const second = await f.storage.openSnapshotJournalSource(identity, request) + expect(second.receipt.requestId).not.toBe(view.receipt.requestId) + expect(BigInt(second.receipt.highWater)).toBeGreaterThan(BigInt(view.receipt.highWater)) + expect((await second.readPage('txLabels')).rows.some(row => row.label === 'after capture')).toBe(true) + await second.close() + } finally { + await view?.close() + await f.close() + } +}) + +test('unfinished generation refuses publication and leaves no receipt; completed migration can be retried', async () => { + const f = await fixture(false) + try { + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('generation') + expect(await f.k('snapshot_journal_receipts')).toEqual([]) + let finished = false + for (let n = 0; n < 80 && !finished; n++) finished = (await copySnapshotJournalBootstrapPage(f.k, 1000000)).complete + await completeSnapshotJournalSqliteGeneration(f.k, request.receiptPolicy) + const view = await f.storage.openSnapshotJournalSource(identity, request) + await view.close() + } finally { + await f.close() + } +}) + +test('an occupied writer refuses promptly without publishing or replaying the independent writer', async () => { + const f = await fixture() + const other = knex(f.k.client.config) + let writer: Knex.Transaction | undefined + try { + writer = await other.transaction() + await writer('tx_labels').where('txLabelId', 1).update({ label: 'uncommitted writer' }) + const start = performance.now() + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toMatchObject({ + code: 'SQLITE_BUSY' + }) + expect(performance.now() - start).toBeLessThan(1000) + expect(await f.k('snapshot_journal_receipts')).toEqual([]) + await writer.commit() + writer = undefined + const view = await f.storage.openSnapshotJournalSource(identity, request) + expect((await view.readPage('txLabels')).rows.some(row => row.label === 'uncommitted writer')).toBe(true) + await view.close() + } finally { + await writer?.rollback() + await other.destroy() + await f.close() + } +}) + +test('cancelled pool setup rejects opening promptly and retains admission until the pending work drains', async () => { + const f = await fixture() + const entered = gate(), + release = gate(), + controller = new AbortController() + const acquire = f.k.client.acquireConnection.bind(f.k.client) + jest.spyOn(f.k.client, 'acquireConnection').mockImplementation(async () => { + entered.resolve() + await release.promise + return await acquire() + }) + try { + const opening = f.storage.openSnapshotJournalSource(identity, request, { + signal: controller.signal + }) + const rejection = expect(opening).rejects.toThrow('cancelled') + await entered.promise + controller.abort() + await rejection + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('already has') + let destroyed = false + const destruction = f.storage.destroy().then(() => { + destroyed = true + }) + await Promise.resolve() + expect(destroyed).toBe(false) + release.resolve() + await destruction + expect(destroyed).toBe(true) + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('destruction') + } finally { + release.resolve() + jest.restoreAllMocks() + await f.close() + } +}) + +test('provider destruction drains an active captured view before closing the foreground pool', async () => { + const f = await fixture() + try { + const view = await f.storage.openSnapshotJournalSource(identity, request) + const detach = jest.spyOn(f.k, 'off') + await f.storage.destroy() + for (const event of ['query', 'query-response', 'query-error']) + expect(detach).toHaveBeenCalledWith(event, expect.any(Function)) + expect(view.isOpen).toBe(false) + await view.closed + await expect(view.readPage('txLabels')).rejects.toThrow('closed') + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('destruction') + } finally { + await f.close() + } +}) + +test('destruction fences a pending source configuration callback before database access', async () => { + const f = await fixture(), + entered = gate(), + release = gate(), + retain = Capture.retainSnapshotJournalCapture, + acquire = jest.spyOn(f.k.client, 'acquireConnection') + let refused: unknown + jest.spyOn(Capture, 'retainSnapshotJournalCapture').mockImplementation((chain, configure, ...rest) => + retain( + chain, + async () => { + entered.resolve() + await release.promise + try { + return await configure() + } catch (error) { + refused = error + throw error + } + }, + ...rest + ) + ) + try { + const opening = f.storage.openSnapshotJournalSource(identity, request), + outcome = opening.then( + value => ({ value }), + error => ({ error }) + ) + await entered.promise + const destruction = f.storage.destroy().then( + value => ({ value }), + error => ({ error }) + ) + release.resolve() + const destroyed = await destruction + expect(refused).toMatchObject({ message: 'Snapshot journal sources are unavailable after destruction begins' }) + expect(destroyed).toHaveProperty('error', refused) + expect(await outcome).toHaveProperty('error') + expect(acquire).not.toHaveBeenCalled() + } finally { + release.resolve() + jest.restoreAllMocks() + await f.close() + } +}) + +test('public input is detached before asynchronous setup and invalid input does not reserve admission', async () => { + const f = await fixture() + try { + await expect(f.storage.awaitSnapshotJournalCaptureCleanup()).resolves.toBeUndefined() + await expect(f.storage.openSnapshotJournalSource('foreign', request)).rejects.toThrow('identityKey') + const input = { ...request, receiptPolicy: { ...request.receiptPolicy } } + const opening = f.storage.openSnapshotJournalSource(identity, input) + input.receiptPolicy.receiptLimit = 1 + input.ceiling = snapshotJournalRevision('1') + const view = await opening + expect(view.receipt.highWater).not.toBe('1') + await view.close() + } finally { + await f.close() + } +}) + +test('source failure is retriable after physical cleanup; an unproved close fences provider capacity', async () => { + const f = await fixture() + try { + await expect(f.storage.openSnapshotJournalSource('02' + '99'.repeat(32), request)).rejects.toThrow( + 'existing wallet profile' + ) + const view = await f.storage.openSnapshotJournalSource(identity, request) + await view.close() + // The cleanup helper is exercised with a failed native close in the backend + // tests. Here assert provider handling of that exact ownership-failure type. + const fault = new SnapshotJournalConnectionCleanupError(new Error('native close failed')) + const config = jest.spyOn(f.k.client, 'acquireConnection').mockRejectedValue(fault) + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toBe(fault) + config.mockRestore() + await expect(f.storage.awaitSnapshotJournalCaptureCleanup()).rejects.toBe(fault) + await expect(f.storage.awaitSnapshotJournalCaptureCleanup()).rejects.toBe(fault) + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('destruction') + await expect(f.storage.destroy()).rejects.toBe(fault) + } finally { + jest.restoreAllMocks() + await f.storage.destroy().catch(() => undefined) + await rm(f.directory, { recursive: true, force: true }) + } +}) + +test('the pinned database type is validated without changing foreground cached settings', async () => { + const f = await fixture() + try { + const cached = f.storage.getSettings() + await f.k('settings').update({ dbtype: 'MySQL' }) + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('generation or profile') + expect(f.storage.getSettings()).toBe(cached) + expect(f.storage.getSettings().dbtype).toBe('SQLite') + expect(await f.k('snapshot_journal_receipts')).toHaveLength(0) + await f.k('settings').update({ dbtype: 'SQLite' }) + const next = await f.storage.openSnapshotJournalSource(identity, request) + expect(next.sourceStorage.dbtype).toBe('SQLite') + await next.close() + } finally { + await f.close() + } +}) + +test('a source failure and owned-provider cleanup failure remain observable together', async () => { + const f = await fixture() + const sourceError = new Error('Backend identity changed') + const cleanupError = new Error('Owned provider did not close') + const destroy = StorageKnex.prototype.destroy + jest.spyOn(Backend, 'bindSnapshotJournalCaptureBackend').mockRejectedValue(sourceError) + jest.spyOn(StorageKnex.prototype, 'destroy').mockImplementation(async function (this: StorageKnex) { + await destroy.call(this) + if (this !== f.storage) throw cleanupError + }) + function causes(value: unknown): unknown[] { + return [ + value, + ...(value instanceof AggregateError ? value.errors.flatMap(causes) : []), + ...(value instanceof Error && value.cause !== undefined ? causes(value.cause) : []) + ] + } + try { + const error = await f.storage.openSnapshotJournalSource(identity, request).catch(value => value) + expect(error).toBeInstanceOf(SnapshotJournalConnectionCleanupError) + expect(causes(error)).toEqual(expect.arrayContaining([sourceError, cleanupError])) + await expect(f.storage.awaitSnapshotJournalCaptureCleanup()).rejects.toBe(error) + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('destruction') + await expect(f.storage.destroy()).rejects.toBe(error) + } finally { + jest.restoreAllMocks() + await f.storage.destroy().catch(() => undefined) + await rm(f.directory, { recursive: true, force: true }) + } +}) + +test.each(['02' + 'ab'.repeat(32) + 'x', 'x02' + 'ab'.repeat(32), '04' + 'ab'.repeat(32), '', undefined, 1])( + 'invalid compressed identity %p refuses synchronously before configuration', + identityKey => { + const configure = jest.fn(async () => undefined) + expect(() => Capture.retainSnapshotJournalCapture('test', configure, identityKey as string, request)).toThrow( + WERR_INVALID_PARAMETER + ) + expect(() => Capture.retainSnapshotJournalCapture('test', configure, identityKey as string, request)).toThrow( + 'identityKey' + ) + expect(configure).not.toHaveBeenCalled() + } +) + +test('zero revision ceiling refuses synchronously before configuration', () => { + const configure = jest.fn(async () => undefined) + expect(() => + Capture.retainSnapshotJournalCapture('test', configure, identity, { + ...request, + ceiling: snapshotJournalRevision('0') + }) + ).toThrow('generation or profile') + expect(configure).not.toHaveBeenCalled() +}) + +test('an unavailable static configuration retains its exact unsupported-provider refusal through cleanup', async () => { + const lifetime = Capture.retainSnapshotJournalCapture('test', async () => undefined, identity, request) + const error = await lifetime.opened.catch(value => value) + expect(error).toMatchObject({ + name: 'WERR_NOT_IMPLEMENTED', + message: 'Snapshot journal capture requires file-backed SQLite WAL or static MySQL' + }) + await expect(lifetime.closed).rejects.toBe(error) +}) + +test.each(['disabled', 'exhausted'] as const)( + 'a native %s event clock commits refusal without a receipt or ordinary source failure', + async state => { + const f = await fixture() + try { + await f + .k('snapshot_journal_clock') + .update(state === 'disabled' ? { enabled: 0, reason: 'capacity-exhausted' } : { revision: request.ceiling }) + const error = await f.storage.openSnapshotJournalSource(identity, request).catch(value => value) + expect(error).toBeInstanceOf(SnapshotResourceLimitError) + expect(error.message).toBe('Snapshot journal event window is exhausted or disabled') + expect(await f.k('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 0, + reason: 'capacity-exhausted' + }) + expect(await f.k('snapshot_journal_receipts')).toEqual([]) + await f.k('tx_labels').where('txLabelId', 1).update({ label: 'ordinary write after exhausted capture' }) + expect((await f.k('tx_labels').where('txLabelId', 1).first()).label).toBe( + 'ordinary write after exhausted capture' + ) + } finally { + await f.close() + } + } +) + +test('a mismatched generation ceiling rolls back its barrier and never publishes a receipt', async () => { + const f = await fixture() + try { + const before = await f.k('snapshot_journal_clock').first() + await expect( + f.storage.openSnapshotJournalSource(identity, { ...request, ceiling: snapshotJournalRevision('1000001') }) + ).rejects.toThrow('generation or profile') + expect(await f.k('snapshot_journal_clock').first()).toEqual(before) + expect(await f.k('snapshot_journal_receipts')).toEqual([]) + } finally { + await f.close() + } +}) + +test.each(['identity', 'chain'] as const)( + 'a pinned %s header mismatch refuses before receipt publication', + async field => { + const f = await fixture(), + read = Archive.readKnexSnapshotArchiveHeader + jest.spyOn(Archive, 'readKnexSnapshotArchiveHeader').mockImplementation(async (...args) => { + const result = await read(...args) + if (field === 'identity') result.header.user.identityKey = '03' + '22'.repeat(32) + else result.header.sourceStorage.chain = 'main' + return result + }) + try { + await expect(f.storage.openSnapshotJournalSource(identity, request)).rejects.toThrow('generation or profile') + expect(await f.k('snapshot_journal_receipts')).toEqual([]) + } finally { + jest.restoreAllMocks() + await f.close() + } + } +) + +test('a receipt failure and both actual transaction rollback failures retain all original causes after native drain', async () => { + const f = await fixture(), + connect = Connections.withSnapshotJournalConnections + const sourceError = new Error('receipt failed'), + rollbackErrors = [new Error('writer rollback failed'), new Error('reader rollback failed')] + jest.spyOn(Receipts, 'recordSnapshotJournalReceipt').mockRejectedValue(sourceError) + jest.spyOn(Connections, 'withSnapshotJournalConnections').mockImplementation(async (...args) => { + for (const [index, owner] of [args[0], args[1]].entries()) { + const transaction = owner.transaction.bind(owner) + Object.defineProperty(owner, 'transaction', { + ...Object.getOwnPropertyDescriptor(owner, 'transaction'), + writable: true + }) + jest.spyOn(owner, 'transaction').mockImplementation(async (...parameters: unknown[]) => { + const trx = await transaction(parameters[0] as Knex.TransactionConfig), + query = trx.client.query.bind(trx.client) + jest.spyOn(trx.client, 'query').mockImplementation(async (...queryParameters: unknown[]) => { + const result: unknown = await query(queryParameters[0], queryParameters[1]) + if (queryParameters[1] === 'ROLLBACK') throw rollbackErrors[index] + return result + }) + return trx + }) + } + return await connect(...args) + }) + function causes(error: unknown): unknown[] { + return [ + error, + ...(error instanceof AggregateError ? error.errors.flatMap(causes) : []), + ...(error instanceof Error && error.cause !== undefined ? causes(error.cause) : []) + ] + } + try { + const error = await f.storage.openSnapshotJournalSource(identity, request).catch(value => value) + expect(error).toBeInstanceOf(SnapshotJournalConnectionCleanupError) + expect(causes(error)).toEqual(expect.arrayContaining([sourceError, ...rollbackErrors])) + expect(await f.k('snapshot_journal_receipts')).toEqual([]) + await expect(f.storage.awaitSnapshotJournalCaptureCleanup()).rejects.toBe(error) + } finally { + jest.restoreAllMocks() + await f.storage.destroy().catch(() => undefined) + await rm(f.directory, { recursive: true, force: true }) + } +}) + +test('successful capture followed by an actual reader rollback failure reports exactly that cleanup cause', async () => { + const f = await fixture(), + connect = Connections.withSnapshotJournalConnections, + rollbackError = new Error('successful source reader rollback failed') + jest.spyOn(Connections, 'withSnapshotJournalConnections').mockImplementation(async (...args) => { + const reader = args[1], + transaction = reader.transaction.bind(reader) + Object.defineProperty(reader, 'transaction', { + ...Object.getOwnPropertyDescriptor(reader, 'transaction'), + writable: true + }) + jest.spyOn(reader, 'transaction').mockImplementation(async (...parameters: unknown[]) => { + const trx = await transaction(parameters[0] as Knex.TransactionConfig), + query = trx.client.query.bind(trx.client) + jest.spyOn(trx.client, 'query').mockImplementation(async (...queryParameters: unknown[]) => { + const result: unknown = await query(queryParameters[0], queryParameters[1]) + if (queryParameters[1] === 'ROLLBACK') throw rollbackError + return result + }) + return trx + }) + return await connect(...args) + }) + const lifetime = Capture.retainSnapshotJournalCapture('test', async () => f.k.client.config, identity, request) + try { + const view = await lifetime.opened + expect((await view.readPage('txLabels')).rows.length).toBeGreaterThan(0) + const error = await lifetime.close().catch(value => value) + expect(error).toBeInstanceOf(SnapshotJournalConnectionCleanupError) + expect(error.cause).toMatchObject({ + message: 'Snapshot capture transactions did not drain', + errors: [rollbackError] + }) + await expect(lifetime.closed).rejects.toBe(error) + expect(await f.k('snapshot_journal_receipts')).toHaveLength(1) + } finally { + await lifetime.close().catch(() => undefined) + jest.restoreAllMocks() + await f.close() + } +}) + +test('successful capture followed by an owned-provider close failure reports exactly that cleanup cause', async () => { + const f = await fixture(), + cleanupError = new Error('successful source owned provider did not close'), + destroy = StorageKnex.prototype.destroy + jest.spyOn(StorageKnex.prototype, 'destroy').mockImplementation(async function (this: StorageKnex) { + await destroy.call(this) + if (this !== f.storage) throw cleanupError + }) + const lifetime = Capture.retainSnapshotJournalCapture('test', async () => f.k.client.config, identity, request) + try { + const view = await lifetime.opened + expect((await view.readPage('txLabels')).rows.length).toBeGreaterThan(0) + const error = await lifetime.close().catch(value => value) + expect(error).toBeInstanceOf(SnapshotJournalConnectionCleanupError) + expect(error.cause).toMatchObject({ + message: 'Snapshot capture owned providers did not close', + errors: [cleanupError] + }) + await expect(lifetime.closed).rejects.toBe(error) + expect(await f.k('snapshot_journal_receipts')).toHaveLength(1) + } finally { + await lifetime.close().catch(() => undefined) + jest.restoreAllMocks() + await f.close() + } +}) + +test('uppercase caller identity selects the existing canonical profile and receipt key', async () => { + const f = await fixture(), + upper = '02' + 'AB'.repeat(32) + try { + await f.k('users').where('userId', 1).update({ identityKey: upper.toLowerCase() }) + const view = await f.storage.openSnapshotJournalSource(upper, request) + try { + expect(view.user.identityKey).toBe(upper.toLowerCase()) + expect(view.receiptBinding.identityKey).toBe(upper.toLowerCase()) + } finally { + await view.close() + } + } finally { + await f.close() + } +}) + +test('MySQL capture configures both reserved pools before transactions, commits before verification and drains its pinned reader', async () => { + // This orchestration double checks exact pool/session/transaction boundaries. + // Native MySQL identity, isolation and thirteen-table data are separate fixtures. + const f = await fixture() + const header = await f.k.transaction(t => Archive.readKnexSnapshotArchiveHeader(f.storage, identity, t)) + header.header.sourceStorage.dbtype = 'MySQL' + const sequence: string[] = [], + pools: Knex[] = [], + handles: Array<{ stream: Duplex }> = [] + const factory = jest.requireActual<{ knex: typeof knex }>('knex'), + create = factory.knex + const createSource = Archive.createKnexSnapshotArchiveSource + jest.spyOn(factory, 'knex').mockImplementation((...args: unknown[]) => { + const config = args[0] as Knex.Config + expect(config.pool).toMatchObject({ min: 0, max: 1 }) + expect(config.acquireConnectionTimeout).toBe(5000) + const role = pools.length === 0 ? 'writer' : 'reader', + owner = create(config) + const handle = { + stream: new Duplex({ + read() {}, + write(_chunk, _encoding, callback) { + callback() + } + }) + } + pools.push(owner) + handles.push(handle) + jest.spyOn(owner.client, 'acquireConnection').mockResolvedValue(handle) + jest.spyOn(owner.client, 'releaseConnection').mockResolvedValue(undefined) + const destroy = owner.client.destroy.bind(owner.client) + jest.spyOn(owner.client, 'destroy').mockImplementation(async () => { + handle.stream.destroy() + await destroy() + }) + const raw = owner.client.raw.bind(owner.client) + jest.spyOn(owner.client, 'raw').mockImplementation((...parameters: unknown[]) => { + expect(role).toBe('reader') + expect(parameters).toEqual(['SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY']) + sequence.push('reader session') + const query = raw('SELECT 1') + query.connection = jest.fn().mockImplementation(async connection => { + expect(connection).toBe(handle) + }) + return query + }) + Object.defineProperty(owner, 'transaction', { + ...Object.getOwnPropertyDescriptor(owner, 'transaction'), + writable: true + }) + jest.spyOn(owner, 'transaction').mockImplementation(async (...parameters: unknown[]) => { + expect(parameters).toEqual([{ connection: handle }]) + sequence.push('begin ' + role) + const completion = gate() + let completed = false + return { + client: owner.client, + isTransaction: true, + isCompleted: () => completed, + executionPromise: completion.promise, + commit: async () => { + expect(completed).toBe(false) + sequence.push('commit ' + role) + completed = true + completion.resolve() + }, + rollback: async () => { + expect(completed).toBe(false) + sequence.push('rollback ' + role) + completed = true + completion.resolve() + } + } as unknown as Knex.Transaction + }) + return owner + }) + jest.spyOn(Backend, 'prepareSnapshotJournalCaptureBackend').mockResolvedValue({ kind: 'mysql' }) + jest.spyOn(Backend, 'bindSnapshotJournalCaptureBackend').mockResolvedValue('b'.repeat(64)) + jest.spyOn(Fence, 'reserveSnapshotJournalCaptureFence').mockImplementation(async () => { + sequence.push('fence') + return snapshotJournalRevision('7') + }) + const generation = { + epoch: '00000000-0000-4000-8000-000000000000', + source: 'c'.repeat(64), + plan: 'd'.repeat(64), + nextObject: 1, + ceiling: request.ceiling, + complete: true, + enabled: true + } + jest + .spyOn(MysqlGeneration, 'readSnapshotJournalMysqlGeneration') + .mockImplementation(async (_view, ceiling, policy) => { + expect(ceiling).toBe(request.ceiling) + expect(policy).toEqual(request.receiptPolicy) + sequence.push('generation') + return generation + }) + jest.spyOn(Archive, 'readKnexSnapshotArchiveHeader').mockImplementation(async () => { + sequence.push('header') + return header + }) + jest.spyOn(ArchiveSql, 'snapshotArchiveDatabaseNow').mockResolvedValue(1000) + jest.spyOn(Receipts, 'recordSnapshotJournalReceipt').mockImplementation(async (_barrier, binding, value) => { + expect(binding.identityKey).toBe(identity) + sequence.push('receipt') + return { ...value, floor: snapshotJournalRevision('0'), binding: Receipts.snapshotJournalReceiptBinding(binding) } + }) + jest.spyOn(Closure, 'assertKnexSnapshotArchiveClosure').mockImplementation(async () => { + sequence.push('verify') + }) + jest.spyOn(Archive, 'createKnexSnapshotArchiveSource').mockImplementation((storage, ...rest) => { + expect(storage.getSettings()).toEqual(header.header.sourceStorage) + expect(storage.getSnapshotSync()).toBeUndefined() + return createSource(storage, ...rest) + }) + let lifetime: Capture.SnapshotJournalCaptureLifetime | undefined + try { + lifetime = Capture.retainSnapshotJournalCapture( + 'test', + async () => ({ + client: 'mysql2', + connection: { database: 'synthetic' }, + pool: { min: 2, max: 10 }, + acquireConnectionTimeout: 50000 + }), + identity, + request + ) + const view = await lifetime.opened + expect(sequence).toEqual([ + 'reader session', + 'begin writer', + 'fence', + 'begin reader', + 'generation', + 'header', + 'receipt', + 'commit writer', + 'verify' + ]) + expect(view.receipt).toMatchObject({ highWater: '7', expiresAt: 601000 }) + expect(view.receiptBinding.schema).toBe( + createHash('sha256').update('snapshot-journal-schema-v1\n').update(header.header.sourceSchema).digest('hex') + ) + await lifetime.close() + expect(sequence.at(-1)).toBe('rollback reader') + expect(handles).toHaveLength(2) + expect(handles.every(handle => handle.stream.closed)).toBe(true) + } finally { + await lifetime?.close().catch(() => undefined) + jest.restoreAllMocks() + await Promise.allSettled(pools.map(owner => owner.destroy())) + await f.close() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCapture.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCapture.ts new file mode 100644 index 000000000..48803386e --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCapture.ts @@ -0,0 +1,295 @@ +import { createHash, randomBytes } from 'node:crypto' +import { knex as createKnex, type Knex } from 'knex' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import type { Chain } from '../../../sdk/types' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER, WERR_NOT_IMPLEMENTED } from '../../../sdk/WERR_errors' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' +import { retainReadSnapshot, type RetainedReadSnapshotOptions } from '../RetainedReadSnapshot' +import { + readKnexSnapshotArchiveHeader, + createKnexSnapshotArchiveSource, + type KnexSnapshotArchiveHeader, + type SnapshotArchiveSource +} from '../archive/KnexSnapshotArchiveSource' +import { assertKnexSnapshotArchiveClosure } from '../archive/KnexSnapshotArchiveClosure' +import { snapshotArchiveDatabaseNow } from '../archive/SnapshotArchiveSql' +import { + snapshotJournalReceiptPolicy, + recordSnapshotJournalReceipt, + type SnapshotJournalReceiptPolicy, + type SnapshotJournalReceipt, + type SnapshotJournalReceiptBinding +} from './SnapshotJournalReceipt' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' +import { readSnapshotJournalSqliteGeneration } from './SnapshotJournalSqliteGeneration' +import { readSnapshotJournalMysqlGeneration } from './SnapshotJournalMysqlGeneration' +import { reserveSnapshotJournalCaptureFence } from './SnapshotJournalCaptureFence' +import { withSnapshotJournalConnections, SnapshotJournalConnectionCleanupError } from './SnapshotJournalConnections' +import { + prepareSnapshotJournalCaptureBackend, + bindSnapshotJournalCaptureBackend, + closeSnapshotJournalCapturePool +} from './SnapshotJournalCaptureBackend' + +export interface SnapshotJournalCaptureRequest { + ceiling: SnapshotJournalRevision + receiptPolicy: SnapshotJournalReceiptPolicy +} +export interface SnapshotJournalSource extends SnapshotArchiveSource { + readonly receipt: SnapshotJournalReceipt + readonly receiptBinding: SnapshotJournalReceiptBinding +} +export interface SnapshotJournalCaptureLifetime { + opened: Promise + closed: Promise + close: () => Promise +} +interface Captured { + header: KnexSnapshotArchiveHeader + receipt: SnapshotJournalReceipt + binding: SnapshotJournalReceiptBinding +} + +function invalid(): never { + throw new WERR_INVALID_OPERATION('Snapshot journal capture generation or profile is unavailable') +} +function ownedConfig(config: Knex.Config): Knex.Config { + const connection = config.connection as object + return { + ...config, + connection: Object.create(Object.getPrototypeOf(connection), Object.getOwnPropertyDescriptors(connection)), + pool: { ...config.pool, min: 0, max: 1 }, + acquireConnectionTimeout: Math.min(config.acquireConnectionTimeout ?? 5000, 5000) + } +} +async function endTransactions(transactions: Knex.Transaction[], failure?: { error: unknown }): Promise { + const settled = await Promise.allSettled( + transactions.map(async trx => { + if (!trx.isCompleted()) { + await trx.rollback() + await trx.executionPromise + } + }) + ) + const failed = settled.filter(result => result.status === 'rejected') + if (failed.length) + throw new SnapshotJournalConnectionCleanupError( + new AggregateError( + [...(failure === undefined ? [] : [failure.error]), ...failed.map(result => result.reason)], + 'Snapshot capture transactions did not drain' + ) + ) +} +async function begin(k: Knex, connection: unknown, transactions: Knex.Transaction[]): Promise { + const trx = await k.transaction({ connection }) + void trx.executionPromise.catch(() => undefined) + transactions.push(trx) + return trx +} +async function commit(trx: Knex.Transaction): Promise { + // Knex's commit query can resolve after a SQL error while executionPromise + // rejects. Both must settle successfully before a receipt authorizes a view. + await trx.commit() + await trx.executionPromise +} +async function prepareReader(writer: Knex, write: unknown, reader: Knex, read: unknown): Promise { + if (reader.client.config.client === 'better-sqlite3') { + await writer.raw('PRAGMA busy_timeout = 0').connection(write) + await reader.raw('PRAGMA query_only = ON').connection(read) + } else { + await reader.raw('SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY').connection(read) + } +} +interface CaptureOptions { + storage: StorageKnex + writer: Knex + write: unknown + read: unknown + backend: string + identityKey: string + request: SnapshotJournalCaptureRequest + assertActive: () => void + hold: (trx: Knex.Transaction, captured: Captured) => Promise +} +async function capture(options: CaptureOptions): Promise { + const { storage, writer, write, read, backend, identityKey, request, assertActive, hold } = options + const reader = storage.knex, + transactions: Knex.Transaction[] = [] + let failure: { error: unknown } | undefined + try { + await prepareReader(writer, write, reader, read) + assertActive() + const barrier = await begin(writer, write, transactions) + assertActive() + const highWater = await reserveSnapshotJournalCaptureFence(barrier) + if (highWater === undefined) { + await commit(barrier) + throw new SnapshotResourceLimitError('Snapshot journal event window is exhausted or disabled') + } + assertActive() + const view = await begin(reader, read, transactions) + assertActive() + const generation = + reader.client.config.client === 'better-sqlite3' + ? await readSnapshotJournalSqliteGeneration(view, request.receiptPolicy, reader.client.config.migrations) + : await readSnapshotJournalMysqlGeneration( + view, + request.ceiling, + request.receiptPolicy, + reader.client.config.migrations + ) + if (!generation.complete || !generation.enabled || generation.ceiling !== request.ceiling) return invalid() + assertActive() + const header = await readKnexSnapshotArchiveHeader(storage, identityKey, view) + if ( + header.header.user.identityKey.toLowerCase() !== identityKey || + header.header.sourceStorage.chain !== storage.chain || + header.header.sourceStorage.dbtype !== (reader.client.config.client === 'better-sqlite3' ? 'SQLite' : 'MySQL') + ) + return invalid() + // This private provider belongs only to this pinned read view. Page size + // preflight needs its database type; never initialize it from a live query + // or change the foreground provider's cached settings. + storage._settings = { ...header.header.sourceStorage } + const binding: SnapshotJournalReceiptBinding = { + backend, + epoch: generation.epoch, + source: generation.source, + schema: createHash('sha256') + .update('snapshot-journal-schema-v1\n') + .update(header.header.sourceSchema) + .digest('hex'), + storageIdentity: header.header.sourceStorage.storageIdentityKey, + identityKey, + userId: header.header.user.userId, + chain: storage.chain + } + assertActive() + const receipt = await recordSnapshotJournalReceipt(barrier, binding, { + requestId: randomBytes(32).toString('hex'), + highWater, + expiresAt: (await snapshotArchiveDatabaseNow(barrier)) + request.receiptPolicy.receiptLifetimeMs + }) + assertActive() + await commit(barrier) + assertActive() + await hold(view, { header, receipt, binding }) + } catch (error) { + failure = { error } + throw error + } finally { + await endTransactions(transactions, failure) + } +} + +/** Internal owned capture. The provider keeps its admission until closed settles. + * This does not register a journal migration or advertise incremental sync. */ +export function retainSnapshotJournalCapture( + chain: Chain, + resolveConfig: () => Promise, + identity: string, + input: SnapshotJournalCaptureRequest, + options: RetainedReadSnapshotOptions = {} +): SnapshotJournalCaptureLifetime { + if (typeof identity !== 'string' || !/^(02|03)[0-9a-fA-F]{64}$/.test(identity)) + throw new WERR_INVALID_PARAMETER('identityKey', 'a compressed public identity key') + const identityKey = identity.toLowerCase() + const request = { + ceiling: snapshotJournalRevision(input.ceiling), + receiptPolicy: snapshotJournalReceiptPolicy(input.receiptPolicy) + } + if (request.ceiling === '0') return invalid() + let storage: StorageKnex | undefined, captured: Captured | undefined + const lifetime = retainReadSnapshot( + async (hold, assertActive) => { + let writer: Knex | undefined, failure: { error: unknown } | undefined + try { + const config = await resolveConfig() + assertActive() + if (config === undefined) + throw new WERR_NOT_IMPLEMENTED('Snapshot journal capture requires file-backed SQLite WAL or static MySQL') + const expected = await prepareSnapshotJournalCaptureBackend(config) + assertActive() + writer = createKnex(ownedConfig(config)) + storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions(chain), + snapshotSync: false, + knex: createKnex(ownedConfig(config)) + }) + await withSnapshotJournalConnections( + writer, + storage.knex, + assertActive, + async (write, read) => { + const backend = await bindSnapshotJournalCaptureBackend(writer!, write, storage!.knex, read, expected) + assertActive() + await capture({ + storage: storage!, + writer: writer!, + write, + read, + backend, + identityKey, + request, + assertActive, + hold: async (trx, value) => { + captured = value + await hold(trx) + } + }) + }, + closeSnapshotJournalCapturePool + ) + } catch (error) { + // Cancellation rejects opening promptly, but physical cleanup still + // controls closed/admission. Preserve unrelated source failures too. + let cancelled = false + try { + assertActive() + } catch (error_) { + cancelled = error_ === error + } + if (!cancelled) failure = { error } + } + const settled = await Promise.allSettled([writer?.destroy(), storage?.destroy()]) + const failed = settled.filter(result => result.status === 'rejected') + if (failed.length) + throw new SnapshotJournalConnectionCleanupError( + new AggregateError( + [...(failure === undefined ? [] : [failure.error]), ...failed.map(result => result.reason)], + 'Snapshot capture owned providers did not close' + ) + ) + if (failure !== undefined) throw failure.error + }, + async trx => { + if (storage === undefined || captured === undefined) return invalid() + const h = captured.header + // Closure verification uses the pinned reader after the writer barrier has + // committed, so foreground writers can progress throughout this work. + await assertKnexSnapshotArchiveClosure( + storage.toDb(trx), + h.header.user.userId, + h.profileIndexes, + h.relationIndexes, + h.certificateIndexes, + h.globalIndexes + ) + }, + { ...options } + ) + const opened = lifetime.opened.then(view => { + if (storage === undefined || captured === undefined || !view.isOpen) return invalid() + return { + ...createKnexSnapshotArchiveSource(storage, view, captured.header), + get isOpen() { + return view.isOpen + }, + receipt: { ...captured.receipt }, + receiptBinding: { ...captured.binding } + } + }) + void opened.catch(() => undefined) + return { opened, closed: lifetime.closed, close: lifetime.close } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.test.ts new file mode 100644 index 000000000..590330ec3 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.test.ts @@ -0,0 +1,431 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rename, rm, writeFile } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { Duplex } from 'node:stream' +import { createHash } from 'node:crypto' +import { + prepareSnapshotJournalCaptureBackend, + bindSnapshotJournalCaptureBackend, + closeSnapshotJournalCapturePool +} from './SnapshotJournalCaptureBackend' +import { withSnapshotJournalConnections, SnapshotJournalConnectionCleanupError } from './SnapshotJournalConnections' + +function gate() { + let resolve!: () => void + const promise = new Promise(yes => { + resolve = yes + }) + return { promise, resolve } +} +function sqlite(filename: string) { + return knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) +} +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'ts569-capture-backend-')) + const filename = join(directory, 'wallet.sqlite') + const writer = sqlite(filename), + reader = sqlite(filename) + await writer.raw('PRAGMA journal_mode = WAL') + return { + directory, + filename, + writer, + reader, + close: async () => { + await Promise.allSettled([writer.destroy(), reader.destroy()]) + await rm(directory, { recursive: true, force: true }) + } + } +} + +test('fresh independent SQLite pools bind one existing file and physically close both reserved native connections', async () => { + const f = await fixture() + try { + const expected = await prepareSnapshotJournalCaptureBackend(f.writer.client.config) + const connections: Array<{ open: boolean }> = [] + const digest = await withSnapshotJournalConnections( + f.writer, + f.reader, + () => undefined, + async (write, read) => { + connections.push(write as { open: boolean }, read as { open: boolean }) + return await bindSnapshotJournalCaptureBackend(f.writer, write, f.reader, read, expected) + }, + closeSnapshotJournalCapturePool + ) + expect(digest).toBe( + createHash('sha256') + .update('snapshot-journal-backend-v1\n') + .update(JSON.stringify(['sqlite', expected.kind === 'sqlite' ? expected.file : undefined])) + .digest('hex') + ) + expect(connections).toHaveLength(2) + expect(connections.map(value => value.open)).toEqual([false, false]) + expect(f.writer.client.pool).toBeUndefined() + expect(f.reader.client.pool).toBeUndefined() + } finally { + await f.close() + } +}) + +test('a replaced SQLite pathname after preparation refuses the source before its barrier callback', async () => { + const f = await fixture() + try { + const expected = await prepareSnapshotJournalCaptureBackend(f.writer.client.config) + await f.writer.destroy() + await rename(f.filename, join(f.directory, 'original.sqlite')) + await writeFile(f.filename, '') + const writer = sqlite(f.filename) + const barrier = jest.fn(async () => undefined) + await expect( + withSnapshotJournalConnections( + writer, + f.reader, + () => undefined, + async (write, read) => { + await bindSnapshotJournalCaptureBackend(writer, write, f.reader, read, expected) + await barrier() + }, + closeSnapshotJournalCapturePool + ) + ).rejects.toThrow('changed') + expect(barrier).not.toHaveBeenCalled() + } finally { + await f.close() + } +}) + +test('different SQLite reader and writer files refuse even when configured within the same directory', async () => { + const f = await fixture(), + other = sqlite(join(f.directory, 'other.sqlite')) + try { + const expected = await prepareSnapshotJournalCaptureBackend(f.writer.client.config) + await expect( + withSnapshotJournalConnections( + f.writer, + other, + () => undefined, + async (write, read) => { + return await bindSnapshotJournalCaptureBackend(f.writer, write, other, read, expected) + }, + closeSnapshotJournalCapturePool + ) + ).rejects.toThrow('changed') + } finally { + await other.destroy() + await f.close() + } +}) + +test.each([ + { client: 'better-sqlite3', connection: { filename: ':memory:' } }, + { client: 'better-sqlite3', connection: { filename: 'file:shared?mode=memory' } }, + { client: 'mysql2', connection: async () => ({ database: 'wallet' }) }, + { client: 'mysql2', connection: { database: 'wallet' }, connectionPool: {} }, + { client: 'pg', connection: { database: 'wallet' } }, + { client: 'mysql', connection: { database: 'wallet' } }, + { client: 'mysql2', connection: { database: 7 } }, + { client: 'better-sqlite3', connection: { filename: '' } }, + { client: 'better-sqlite3', connection: { filename: 1 } }, + { client: 'mysql2', connection: 'synthetic' }, + { client: 'mysql2', connection: null } +])('unsupported backend configuration refuses before constructing an owned pool: %#', async config => { + await expect(prepareSnapshotJournalCaptureBackend(config as Knex.Config)).rejects.toThrow('unavailable') +}) + +let nativeConnectionId = 0 +function mysqlIdentity( + serverUuid: unknown, + databaseName: unknown, + connectionId: unknown = String(++nativeConnectionId), + rows?: unknown[] +) { + const owner = knex({ + client: 'mysql2', + connection: { database: 'synthetic' }, + pool: { min: 0, max: 1 } + }) + const query = owner.raw('SELECT 1') + query.connection = jest.fn().mockReturnValue(Promise.resolve([rows ?? [{ serverUuid, databaseName, connectionId }]])) + jest.spyOn(owner.client, 'raw').mockReturnValue(query) + return owner +} +const uuid = 'aabbccdd-1234-4321-aabb-112233445566' + +test('MySQL binds the actual server and database returned by both exact reserved connections', async () => { + const writer = mysqlIdentity(uuid.toUpperCase(), 'synthetic'), + reader = mysqlIdentity(uuid, 'synthetic') + try { + const expected = await prepareSnapshotJournalCaptureBackend(writer.client.config) + const writerConnection = {}, + readerConnection = {} + expect(await bindSnapshotJournalCaptureBackend(writer, writerConnection, reader, readerConnection, expected)).toBe( + createHash('sha256') + .update('snapshot-journal-backend-v1\n') + .update(JSON.stringify(['mysql', { serverUuid: uuid, database: 'synthetic' }])) + .digest('hex') + ) + const sql = + 'SELECT @@server_uuid AS serverUuid, DATABASE() AS databaseName, CAST(CONNECTION_ID() AS CHAR) AS connectionId' + expect(writer.client.raw).toHaveBeenCalledWith(sql) + expect(reader.client.raw).toHaveBeenCalledWith(sql) + expect(writer.client.raw(sql).connection).toHaveBeenCalledWith(writerConnection) + expect(reader.client.raw(sql).connection).toHaveBeenCalledWith(readerConnection) + } finally { + await writer.destroy() + await reader.destroy() + jest.restoreAllMocks() + } +}) + +test.each([ + ['different server', 'ffeeddcc-1234-4321-aabb-112233445566', 'synthetic'], + ['different database', uuid, 'other'], + ['invalid uuid', 'g'.repeat(36), 'synthetic'], + ['missing uuid', undefined, 'synthetic'], + ['uuid prefix', 'x' + uuid, 'synthetic'], + ['uuid suffix', uuid + 'x', 'synthetic'], + ['nonstring database', uuid, 7], + ['empty database', uuid, ''], + ['oversized database', uuid, 'é'.repeat(129)] +])('MySQL actual identity refuses %s', async (_label, serverUuid, databaseName) => { + const writer = mysqlIdentity(uuid, 'synthetic'), + reader = mysqlIdentity(serverUuid, databaseName) + try { + await expect(bindSnapshotJournalCaptureBackend(writer, {}, reader, {}, { kind: 'mysql' })).rejects.toThrow( + 'changed' + ) + } finally { + await writer.destroy() + await reader.destroy() + jest.restoreAllMocks() + } +}) + +test('graceful MySQL quit does not establish physical cleanup until the socket close event', async () => { + const owner = knex({ + client: 'mysql2', + connection: { database: 'synthetic' }, + pool: { min: 0, max: 1 } + }) + const stream = new Duplex({ + read() {}, + write(_chunk, _encoding, callback) { + callback() + } + }) + const entered = gate(), + allow = gate() + jest.spyOn(owner.client, 'destroy').mockImplementation(async () => { + entered.resolve() + await allow.promise + }) + jest.spyOn(owner.client, 'releaseConnection').mockResolvedValue(undefined) + let closed = false + try { + const cleanup = closeSnapshotJournalCapturePool(owner, { stream }).then(() => { + closed = true + }) + await entered.promise + allow.resolve() + await Promise.resolve() + await Promise.resolve() + expect(closed).toBe(false) + expect(stream.listenerCount('close')).toBe(1) + stream.destroy() + await cleanup + expect(stream.closed).toBe(true) + expect(closed).toBe(true) + expect(stream.listenerCount('close')).toBe(0) + } finally { + allow.resolve() + stream.destroy() + jest.restoreAllMocks() + await owner.destroy() + } +}) + +test('physical cleanup refusal is typed and retains both source and native release failures', async () => { + const f = await fixture() + const release = f.writer.client.releaseConnection.bind(f.writer.client) + let held: unknown + jest.spyOn(f.writer.client, 'releaseConnection').mockRejectedValue(new Error('release failed')) + try { + const operation = withSnapshotJournalConnections( + f.writer, + f.reader, + () => undefined, + async write => { + held = write + throw new Error('source failed') + }, + async (owner, connection) => { + if (owner === f.writer) { + await owner.client.releaseConnection(connection) + } else await closeSnapshotJournalCapturePool(owner, connection) + } + ) + await expect(operation).rejects.toBeInstanceOf(SnapshotJournalConnectionCleanupError) + await expect(operation).rejects.toMatchObject({ + cause: { + errors: [ + expect.objectContaining({ message: 'source failed' }), + expect.objectContaining({ message: 'release failed' }) + ] + } + }) + } finally { + jest.restoreAllMocks() + if (held !== undefined) await release(held) + await f.close() + } +}) + +test.each(['0', '01', 1, '18446744073709551616', '12x', 'x12', '1x'])( + 'MySQL refuses invalid actual native connection identity %p', + async id => { + const writer = mysqlIdentity(uuid, 'synthetic', id), + reader = mysqlIdentity(uuid, 'synthetic') + try { + const expected = await prepareSnapshotJournalCaptureBackend(writer.client.config) + await expect(bindSnapshotJournalCaptureBackend(writer, {}, reader, {}, expected)).rejects.toThrow('changed') + } finally { + await writer.destroy() + await reader.destroy() + jest.restoreAllMocks() + } + } +) + +test('distinct pool objects cannot publish two handles addressing the same native MySQL connection', async () => { + const writer = mysqlIdentity(uuid, 'synthetic', '42'), + reader = mysqlIdentity(uuid, 'synthetic', '42') + try { + const expected = await prepareSnapshotJournalCaptureBackend(writer.client.config) + await expect(bindSnapshotJournalCaptureBackend(writer, {}, reader, {}, expected)).rejects.toThrow('changed') + } finally { + await writer.destroy() + await reader.destroy() + jest.restoreAllMocks() + } +}) + +test('backend preparation rejects a directory even when its pathname exists', async () => { + const f = await fixture() + try { + await expect( + prepareSnapshotJournalCaptureBackend({ client: 'better-sqlite3', connection: { filename: f.directory } }) + ).rejects.toThrow('unavailable') + } finally { + await f.close() + } +}) + +test.each([ + { rows: [] }, + { + rows: [ + { serverUuid: uuid, databaseName: 'synthetic', connectionId: '1' }, + { serverUuid: uuid, databaseName: 'synthetic', connectionId: '2' } + ] + } +])('MySQL refuses a non-singleton native identity result %#', async ({ rows }) => { + const writer = mysqlIdentity(uuid, 'synthetic', '3', rows), + reader = mysqlIdentity(uuid, 'synthetic', '4') + try { + await expect(bindSnapshotJournalCaptureBackend(writer, {}, reader, {}, { kind: 'mysql' })).rejects.toThrow( + 'changed' + ) + } finally { + jest.restoreAllMocks() + await Promise.all([writer.destroy(), reader.destroy()]) + } +}) + +test('MySQL accepts exact 256 UTF8 database bytes and the unsigned64 connection boundary', async () => { + const database = 'é'.repeat(128) + const writer = mysqlIdentity(uuid, database, '18446744073709551615'), + reader = mysqlIdentity(uuid, database, '1') + try { + expect(await bindSnapshotJournalCaptureBackend(writer, {}, reader, {}, { kind: 'mysql' })).toBe( + createHash('sha256') + .update('snapshot-journal-backend-v1\n') + .update(JSON.stringify(['mysql', { serverUuid: uuid, database }])) + .digest('hex') + ) + } finally { + jest.restoreAllMocks() + await Promise.all([writer.destroy(), reader.destroy()]) + } +}) + +test('native pool destruction failures retain every cause and remove the close listener', async () => { + const owner = mysqlIdentity(uuid, 'synthetic') + const stream = new Duplex({ + read() {}, + write(_chunk, _encoding, callback) { + callback() + } + }) + const destroyFailure = new Error('destroy failed'), + releaseFailure = new Error('release failed') + jest.spyOn(owner.client, 'destroy').mockRejectedValue(destroyFailure) + jest.spyOn(owner.client, 'releaseConnection').mockRejectedValue(releaseFailure) + try { + await expect(closeSnapshotJournalCapturePool(owner, { stream })).rejects.toMatchObject({ + message: 'Snapshot capture pool did not close', + errors: [destroyFailure, releaseFailure] + }) + expect(stream.listenerCount('close')).toBe(0) + } finally { + stream.destroy() + jest.restoreAllMocks() + await owner.destroy() + } +}) + +test.each(['better-sqlite3', 'mysql2'])( + 'successful pool calls cannot claim an unclosed %s native handle', + async client => { + const owner = knex({ + client, + connection: client === 'better-sqlite3' ? { filename: ':memory:' } : { database: 'synthetic' }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) + jest.spyOn(owner.client, 'destroy').mockResolvedValue(undefined) + jest.spyOn(owner.client, 'releaseConnection').mockResolvedValue(undefined) + try { + await expect(closeSnapshotJournalCapturePool(owner, { open: true })).rejects.toThrow( + 'Snapshot capture native connection did not close' + ) + } finally { + jest.restoreAllMocks() + await owner.destroy() + } + } +) + +test('fresh native connection IDs do not change the durable backend binding', async () => { + const pools = [ + mysqlIdentity(uuid, 'synthetic', '1'), + mysqlIdentity(uuid, 'synthetic', '2'), + mysqlIdentity(uuid, 'synthetic', '3'), + mysqlIdentity(uuid, 'synthetic', '4') + ] + try { + const expected = await prepareSnapshotJournalCaptureBackend(pools[0].client.config) + expect(await bindSnapshotJournalCaptureBackend(pools[0], {}, pools[1], {}, expected)).toBe( + await bindSnapshotJournalCaptureBackend(pools[2], {}, pools[3], {}, expected) + ) + } finally { + await Promise.all(pools.map(pool => pool.destroy())) + jest.restoreAllMocks() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.ts new file mode 100644 index 000000000..481d8881b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.ts @@ -0,0 +1,135 @@ +import type { Knex } from 'knex' +import { createHash } from 'node:crypto' +import { lstat, realpath } from 'node:fs/promises' +import type { Duplex } from 'node:stream' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' + +interface FileIdentity { + path: string + device: string + inode: string +} +export type SnapshotJournalCaptureBackend = { kind: 'sqlite'; file: FileIdentity } | { kind: 'mysql' } + +function unavailable(): never { + throw new WERR_INVALID_OPERATION('Snapshot journal capture backend is unavailable or changed') +} +async function fileIdentity(filename: string): Promise { + const path = await realpath(filename) + const info = await lstat(path, { bigint: true }) + if (!info.isFile()) return unavailable() + return { path, device: info.dev.toString(), inode: info.ino.toString() } +} + +/** Call before constructing either fresh owned pool; existing open SQLite handles + * cannot establish file identity by rechecking only their configured pathname. */ +export async function prepareSnapshotJournalCaptureBackend( + config: Knex.Config +): Promise { + const connection: unknown = config.connection + if ( + connection === null || + typeof connection !== 'object' || + ('connectionPool' in config && config.connectionPool != null) + ) + return unavailable() + if (config.client === 'better-sqlite3') { + const filename = (connection as Knex.Sqlite3ConnectionConfig).filename + if (typeof filename !== 'string' || !filename || filename === ':memory:' || filename.startsWith('file:')) + return unavailable() + return { kind: 'sqlite', file: await fileIdentity(filename) } + } + if (config.client === 'mysql2' && 'database' in connection && typeof connection.database === 'string') + return { kind: 'mysql' } + return unavailable() +} +interface ActualBackendIdentity { + backend: unknown + connectionId?: string +} +async function actualIdentity( + k: Knex, + connection: unknown, + expected: SnapshotJournalCaptureBackend +): Promise { + if (expected.kind === 'sqlite') { + const rows: Array<{ name: string; file: string }> = await k.raw('PRAGMA database_list').connection(connection) + const main = rows.filter(row => row.name === 'main') + if (main.length !== 1 || typeof main[0].file !== 'string') return unavailable() + const actual = await fileIdentity(main[0].file) + if (JSON.stringify(actual) !== JSON.stringify(expected.file)) return unavailable() + return { backend: actual } + } + const [rows]: Array> = await k + .raw( + 'SELECT @@server_uuid AS serverUuid, DATABASE() AS databaseName, CAST(CONNECTION_ID() AS CHAR) AS connectionId' + ) + .connection(connection) + const row = rows.at(0) + if ( + rows.length !== 1 || + typeof row?.serverUuid !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(row.serverUuid) || + typeof row.databaseName !== 'string' || + !row.databaseName || + Buffer.byteLength(row.databaseName, 'utf8') > 256 || + typeof row.connectionId !== 'string' || + !/^[1-9]\d{0,19}$/.test(row.connectionId) || + BigInt(row.connectionId) > 18446744073709551615n + ) + return unavailable() + return { + backend: { serverUuid: row.serverUuid.toLowerCase(), database: row.databaseName }, + connectionId: row.connectionId + } +} + +/** Both actual native connections must address the same backend before a barrier. */ +export async function bindSnapshotJournalCaptureBackend( + writer: Knex, + writerConnection: unknown, + reader: Knex, + readerConnection: unknown, + expected: SnapshotJournalCaptureBackend +): Promise { + const write = await actualIdentity(writer, writerConnection, expected) + const read = await actualIdentity(reader, readerConnection, expected) + if ( + JSON.stringify(write.backend) !== JSON.stringify(read.backend) || + (expected.kind === 'mysql' && write.connectionId === read.connectionId) + ) + return unavailable() + return createHash('sha256') + .update('snapshot-journal-backend-v1\n') + .update(JSON.stringify([expected.kind, write.backend])) + .digest('hex') +} + +/** The pool is owned exclusively by one capture. Destroy it while releasing its + * reserved slot, and wait for native closure before reporting physical cleanup. */ +export async function closeSnapshotJournalCapturePool(k: Knex, value: unknown): Promise { + const connection = value as { open?: boolean; stream?: Duplex } + const stream = connection.stream + let listener: (() => void) | undefined + const nativeClose = + stream === undefined || stream.closed + ? undefined + : new Promise(resolve => { + listener = resolve + stream.once('close', resolve) + }) + try { + const results = await Promise.allSettled([k.destroy(), k.client.releaseConnection(value)]) + const failed = results.filter(result => result.status === 'rejected') + if (failed.length) + throw new AggregateError( + failed.map(result => result.reason), + 'Snapshot capture pool did not close' + ) + await nativeClose + if (k.client.config.client === 'better-sqlite3' ? connection.open !== false : stream?.closed !== true) + throw new Error('Snapshot capture native connection did not close') + } finally { + if (listener !== undefined) stream!.removeListener('close', listener) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureFence.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureFence.test.ts new file mode 100644 index 000000000..7ef2c54e4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureFence.test.ts @@ -0,0 +1,327 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { reserveSnapshotJournalCaptureFence } from './SnapshotJournalCaptureFence' +import { installSnapshotJournalSqliteClock, SNAPSHOT_JOURNAL_SQLITE_ADVANCE } from './SnapshotJournalSqliteClock' +import { snapshotJournalRevision, MAX_SNAPSHOT_JOURNAL_REVISION } from './SnapshotJournalRevision' + +async function fixture(ceiling = '9223372036854775807') { + const directory = await mkdtemp(join(tmpdir(), 'ts569-capture-fence-')) + const filename = join(directory, 'wallet.sqlite') + const open = () => + knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) + const writer = open(), + barrier = open(), + reader = open() + await writer.raw('PRAGMA journal_mode=WAL') + await barrier.raw('PRAGMA busy_timeout=0') + await installSnapshotJournalSqliteClock(writer, snapshotJournalRevision(ceiling)) + await writer.raw('CREATE TABLE source(id INTEGER PRIMARY KEY,value INTEGER NOT NULL)') + await writer('source').insert({ id: 1, value: 0 }) + await writer.raw( + 'CREATE TRIGGER observe_source AFTER UPDATE ON source BEGIN ' + SNAPSHOT_JOURNAL_SQLITE_ADVANCE + ' END' + ) + return { + writer, + barrier, + reader, + close: async () => { + await reader.destroy() + await barrier.destroy() + await writer.destroy() + await rm(directory, { recursive: true, force: true }) + } + } +} + +test('a reserved global fence separates independent committed writers and leaves the pinned view immutable', async () => { + const f = await fixture() + let view: Knex.Transaction | undefined + try { + await f.writer('source').update({ value: 1 }) + let fence: string | undefined + await f.barrier.transaction(async t => { + fence = await reserveSnapshotJournalCaptureFence(t) + expect(fence).toBe('2') + view = await f.reader.transaction() + expect(await view('source').first()).toEqual({ id: 1, value: 1 }) + expect((await view('snapshot_journal_clock').first()).revision).toBe(1) + }) + await f.writer('source').update({ value: 2 }) + expect((await f.writer('snapshot_journal_clock').first()).revision).toBe(3) + expect(await view!('source').first()).toEqual({ id: 1, value: 1 }) + expect(fence).toBe('2') + } finally { + await view?.rollback() + await f.close() + } +}) + +test('occupied writer barrier refuses promptly and does not consume a position or replay a writer', async () => { + const f = await fixture() + let writer: Knex.Transaction | undefined + try { + writer = await f.writer.transaction() + await writer('source').update({ value: 1 }) + const start = performance.now() + await expect(f.barrier.transaction(reserveSnapshotJournalCaptureFence)).rejects.toMatchObject({ + code: 'SQLITE_BUSY' + }) + expect(performance.now() - start).toBeLessThan(1000) + await writer.commit() + writer = undefined + expect((await f.writer('snapshot_journal_clock').first()).revision).toBe(1) + expect(await f.barrier.transaction(reserveSnapshotJournalCaptureFence)).toBe('2') + } finally { + await writer?.rollback() + await f.close() + } +}) + +test('a rolled-back fence cannot publish and its position is reused only before any receipt commit', async () => { + const f = await fixture() + try { + await expect( + f.barrier.transaction(async t => { + expect(await reserveSnapshotJournalCaptureFence(t)).toBe('1') + throw Error('rollback') + }) + ).rejects.toThrow('rollback') + expect((await f.writer('snapshot_journal_clock').first()).revision).toBe(0) + expect(await f.barrier.transaction(reserveSnapshotJournalCaptureFence)).toBe('1') + } finally { + await f.close() + } +}) + +test.each(['2', MAX_SNAPSHOT_JOURNAL_REVISION])( + 'capture exhaustion at %s persists disablement only on commit', + async ceiling => { + const f = await fixture(ceiling) + try { + await f.writer.raw('UPDATE snapshot_journal_clock SET revision=CAST(? AS INTEGER)', [ + (BigInt(ceiling) - 1n).toString() + ]) + expect(await f.barrier.transaction(reserveSnapshotJournalCaptureFence)).toBe(ceiling) + expect(await f.barrier.transaction(reserveSnapshotJournalCaptureFence)).toBeUndefined() + expect(await f.writer('snapshot_journal_clock').first('enabled', 'reason')).toEqual({ + enabled: 0, + reason: ceiling === MAX_SNAPSHOT_JOURNAL_REVISION ? 'revision-exhausted' : 'capacity-exhausted' + }) + await f.writer('source').update({ value: 1 }) + expect(await f.writer('source').first()).toEqual({ id: 1, value: 1 }) + } finally { + await f.close() + } + } +) + +test('capture refuses ambient nontransaction, unsupported driver and busy-waiting admission', async () => { + const f = await fixture() + try { + await expect(reserveSnapshotJournalCaptureFence(f.barrier)).rejects.toThrow('Invalid snapshot journal capture') + await expect( + reserveSnapshotJournalCaptureFence({ + isTransaction: true, + client: { config: { client: 'pg' } } + } as unknown as Knex) + ).rejects.toThrow('Invalid snapshot journal capture') + await f.barrier.raw('PRAGMA busy_timeout=1') + await expect(f.barrier.transaction(reserveSnapshotJournalCaptureFence)).rejects.toThrow( + 'Invalid snapshot journal capture' + ) + expect((await f.writer('snapshot_journal_clock').first()).revision).toBe(0) + } finally { + await f.close() + } +}) + +test.each(['missing-clock', 'invalid-enabled', 'zero-ceiling', 'revision-over-ceiling', 'disabled'] as const)( + 'SQLite capture reads the exact persisted %s state before allocation', + async state => { + const f = await fixture() + try { + if (state === 'missing-clock') await f.writer('snapshot_journal_clock').delete() + else { + await f.writer.raw('PRAGMA ignore_check_constraints=ON') + if (state === 'invalid-enabled') await f.writer('snapshot_journal_clock').update({ enabled: 2 }) + else if (state === 'zero-ceiling') await f.writer('snapshot_journal_clock').update({ ceiling: 0 }) + else if (state === 'revision-over-ceiling') + await f.writer('snapshot_journal_clock').update({ revision: 2, ceiling: 1 }) + else await f.writer('snapshot_journal_clock').update({ enabled: 0, reason: 'capacity-exhausted' }) + await f.writer.raw('PRAGMA ignore_check_constraints=OFF') + await f.barrier.raw('PRAGMA ignore_check_constraints=ON') + } + if (state === 'disabled') expect(await f.barrier.transaction(reserveSnapshotJournalCaptureFence)).toBeUndefined() + else + await expect(f.barrier.transaction(reserveSnapshotJournalCaptureFence)).rejects.toThrow( + 'Invalid snapshot journal capture barrier or clock' + ) + } finally { + await f.close() + } + } +) + +test.each(['no-clock', 'invalid-enabled', 'wrong-increment', 'over-ceiling'] as const)( + 'SQLite capture refuses %s returned after the actual clock advance', + async fault => { + const f = await fixture('10') + try { + await f.barrier.raw('PRAGMA ignore_check_constraints=ON') + let effect: string + if (fault === 'no-clock') effect = 'DELETE FROM snapshot_journal_clock' + else if (fault === 'invalid-enabled') effect = 'UPDATE snapshot_journal_clock SET enabled=2' + else effect = 'UPDATE snapshot_journal_clock SET revision=' + (fault === 'wrong-increment' ? '0' : '11') + await f.writer.raw( + 'CREATE TRIGGER corrupt_capture_clock AFTER UPDATE ON snapshot_journal_clock WHEN NEW.revision=1 BEGIN ' + + effect + + '; END' + ) + await expect(f.barrier.transaction(reserveSnapshotJournalCaptureFence)).rejects.toThrow( + 'Invalid snapshot journal capture barrier or clock' + ) + expect((await f.writer('snapshot_journal_clock').first()).revision).toBe(0) + } finally { + await f.close() + } + } +) + +test('SQLite capture supports the sqlite3 driver identity', async () => { + const f = await fixture() + try { + await f.barrier.transaction(async t => { + t.client.config.client = 'sqlite3' + expect(await reserveSnapshotJournalCaptureFence(t)).toBe('1') + }) + } finally { + await f.close() + } +}) + +test('SQLite capture refuses native non-WAL admission before clock access', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) + try { + await k.raw('PRAGMA busy_timeout=0') + await expect(k.transaction(reserveSnapshotJournalCaptureFence)).rejects.toThrow( + 'Invalid snapshot journal capture barrier or clock' + ) + } finally { + await k.destroy() + } +}) + +/** Real Knex MySQL compilation and response decoding, with only the native + * transport replaced. Actual isolation/locking is covered by the native cohort. */ +function mysqlTransport(ceiling: string, allocated: unknown, disabled = false, missing = false) { + const owner = knex({ client: 'mysql2', connection: { database: 'synthetic' }, pool: { min: 0, max: 1 } }) + const queries: Array<{ sql: string; bindings: unknown[] }> = [] + const invalidations: unknown[][] = [] + jest.spyOn(owner.client, 'acquireConnection').mockResolvedValue({}) + jest.spyOn(owner.client, 'releaseConnection').mockResolvedValue(undefined) + jest.spyOn(owner.client, 'query').mockImplementation(async (...args: unknown[]) => { + const query = args[1] as { sql: string; method: string; bindings: unknown[] } + queries.push({ sql: query.sql, bindings: query.bindings ?? [] }) + let rows: unknown + if ( + query.sql === + 'select CAST(`ceiling` AS CHAR) as `ceiling` from `snapshot_journal_clock` where `id` = ? limit ? for update nowait' + ) + rows = missing ? [] : [{ ceiling }] + else if (query.sql === 'select `id` from `snapshot_journal_invalid` where `id` = ? limit ? for update nowait') + rows = disabled ? [{ id: 1 }] : [] + else if (query.sql === 'insert into `snapshot_journal_events` () values ()') rows = { insertId: allocated } + else if (query.sql === 'SELECT CAST(LAST_INSERT_ID() AS CHAR) revision') rows = [{ revision: allocated }] + else if (query.sql === 'delete from `snapshot_journal_events` where `revision` = ?') rows = { affectedRows: 1 } + else if (query.sql === 'insert ignore into `snapshot_journal_invalid` (`id`, `reason`) values (?, ?)') { + invalidations.push(query.bindings) + rows = { insertId: 1 } + } else throw new Error('Unexpected capture SQL: ' + query.sql) + return { ...query, response: [rows, []] } + }) + return { + k: Object.assign(owner, { isTransaction: true }), + queries, + invalidations, + close: async () => { + jest.restoreAllMocks() + await owner.destroy() + } + } +} + +test.each(['mysql2', 'mysql'])('capture uses exact current NOWAIT reads and decimal allocation on %s', async client => { + const f = mysqlTransport(MAX_SNAPSHOT_JOURNAL_REVISION, '9007199254740993') + try { + f.k.client.config.client = client + expect(await reserveSnapshotJournalCaptureFence(f.k)).toBe('9007199254740993') + expect(f.queries.map(query => query.bindings)).toEqual([[1, 1], [1, 1], [], [], ['9007199254740993']]) + expect(f.invalidations).toEqual([]) + } finally { + await f.close() + } +}) + +test.each([ + ['2', '2', undefined], + ['2', '3', 'capacity-exhausted'], + [MAX_SNAPSHOT_JOURNAL_REVISION, '9223372036854775808', 'revision-exhausted'] +] as const)('MySQL allocation %s/%s retains exact exhaustion identity', async (ceiling, allocated, reason) => { + const f = mysqlTransport(ceiling, allocated) + try { + expect(await reserveSnapshotJournalCaptureFence(f.k)).toBe(reason ? undefined : allocated) + expect(f.invalidations).toEqual(reason ? [[1, reason]] : []) + expect(f.queries[4]).toEqual({ + sql: 'delete from `snapshot_journal_events` where `revision` = ?', + bindings: [allocated] + }) + } finally { + await f.close() + } +}) + +test.each(['0', '01', '', 'x1', '1x', 1, undefined])( + 'MySQL refuses malformed native allocation %p before deleting an event', + async allocated => { + const f = mysqlTransport('10', allocated) + try { + await expect(reserveSnapshotJournalCaptureFence(f.k)).rejects.toThrow( + 'Invalid snapshot journal capture barrier or clock' + ) + expect(f.queries).toHaveLength(4) + expect(f.invalidations).toEqual([]) + } finally { + await f.close() + } + } +) + +test.each(['disabled', 'missing', 'zero-ceiling'] as const)( + 'MySQL %s state stops before event allocation', + async state => { + const f = mysqlTransport(state === 'zero-ceiling' ? '0' : '10', '1', state === 'disabled', state === 'missing') + try { + if (state === 'disabled') expect(await reserveSnapshotJournalCaptureFence(f.k)).toBeUndefined() + else + await expect(reserveSnapshotJournalCaptureFence(f.k)).rejects.toThrow( + 'Invalid snapshot journal capture barrier or clock' + ) + expect(f.queries).toHaveLength(state === 'disabled' ? 2 : 1) + expect(f.invalidations).toEqual([]) + } finally { + await f.close() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureFence.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureFence.ts new file mode 100644 index 000000000..45a6eb1de --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCaptureFence.ts @@ -0,0 +1,98 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { + compareSnapshotJournalRevisions, + MAX_SNAPSHOT_JOURNAL_REVISION, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' +import { snapshotJournalRevisionText } from './SnapshotJournalRevisionSql' +import { SNAPSHOT_JOURNAL_SQLITE_ADVANCE } from './SnapshotJournalSqliteClock' + +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid snapshot journal capture barrier or clock') +} + +async function sqliteFence(k: Knex): Promise { + const timeouts: Array<{ timeout: number }> = await k.raw('PRAGMA busy_timeout') + const modes: Array<{ journal_mode: string }> = await k.raw('PRAGMA journal_mode') + if (timeouts.length !== 1 || timeouts[0].timeout !== 0 || modes.length !== 1 || modes[0].journal_mode !== 'wal') + return invalid() + // Obtain the write reservation before any read can establish an old view. + if ( + (await k('snapshot_journal_clock') + .where('id', 1) + .update({ revision: k.ref('revision') })) !== 1 + ) + return invalid() + const before = await k('snapshot_journal_clock') + .where('id', 1) + .select( + 'enabled', + { revision: snapshotJournalRevisionText(k, 'revision') }, + { ceiling: snapshotJournalRevisionText(k, 'ceiling') } + ) + .first() + if (!before || ![0, 1].includes(before.enabled)) return invalid() + const revision = snapshotJournalRevision(before.revision), + ceiling = snapshotJournalRevision(before.ceiling) + if (ceiling === '0' || compareSnapshotJournalRevisions(revision, ceiling) > 0) return invalid() + if (before.enabled === 0) return undefined + await k.raw(SNAPSHOT_JOURNAL_SQLITE_ADVANCE) + const after = await k('snapshot_journal_clock') + .where('id', 1) + .select('enabled', { revision: snapshotJournalRevisionText(k, 'revision') }) + .first() + if (!after || ![0, 1].includes(after.enabled)) return invalid() + const value = snapshotJournalRevision(after.revision) + if (after.enabled === 0) return undefined + if (BigInt(value) !== BigInt(revision) + 1n || compareSnapshotJournalRevisions(value, ceiling) > 0) return invalid() + return value +} + +async function mysqlFence(k: Knex): Promise { + const clock = await k('snapshot_journal_clock') + .where('id', 1) + .select({ ceiling: snapshotJournalRevisionText(k, 'ceiling') }) + .forUpdate() + .noWait() + .first() + if (!clock) return invalid() + const ceiling = snapshotJournalRevision(clock.ceiling) + if (ceiling === '0') return invalid() + // Current reads are required even if the caller earlier established an RR view. + if (await k('snapshot_journal_invalid').where('id', 1).forUpdate().noWait().first('id')) return undefined + await k('snapshot_journal_events').insert({}) + const [[allocated]]: Array> = await k.raw( + 'SELECT CAST(LAST_INSERT_ID() AS CHAR) revision' + ) + if (typeof allocated?.revision !== 'string' || !/^[1-9]\d{0,19}$/.test(allocated.revision)) return invalid() + const value = BigInt(allocated.revision) + await k('snapshot_journal_events').where('revision', allocated.revision).delete() + if (value > BigInt(ceiling)) { + await k('snapshot_journal_invalid') + .insert({ + id: 1, + reason: value > BigInt(MAX_SNAPSHOT_JOURNAL_REVISION) ? 'revision-exhausted' : 'capacity-exhausted' + }) + .onConflict('id') + .ignore() + return undefined + } + return snapshotJournalRevision(allocated.revision) +} + +/** Internal primitive for an owned, validated, complete generation. The caller + * must reserve both native connections before beginning this fresh transaction. + * The exclusive barrier remains held until its transaction ends. Pin the separate + * reader, record its receipt, and commit before publishing. Undefined means the + * event window is disabled/exhausted: commit its invalidation without publishing. + * This consumes one event position; it does not advance a retention floor. + */ +export async function reserveSnapshotJournalCaptureFence(k: Knex): Promise { + if (!k.isTransaction) return invalid() + const client = k.client.config.client + if (client === 'better-sqlite3' || client === 'sqlite3') return await sqliteFence(k) + if (client === 'mysql' || client === 'mysql2') return await mysqlFence(k) + return invalid() +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCollection.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCollection.test.ts new file mode 100644 index 000000000..92183b62f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCollection.test.ts @@ -0,0 +1,519 @@ +import { knex, type Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { mkdtemp, rm } from 'node:fs/promises' +import { join } from 'node:path' +import { tmpdir } from 'node:os' +import { SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL } from './SnapshotJournalSqliteClock' +import { SNAPSHOT_JOURNAL_SQLITE_GENERATION_DDL } from './SnapshotJournalSqliteGeneration' +import { SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL } from './SnapshotJournalSqliteObservers' +import { snapshotJournalReceiptDdl } from './SnapshotJournalReceipt' +import { snapshotJournalRevision } from './SnapshotJournalRevision' +import { + collectSnapshotJournalTombstones, + snapshotJournalCollectionQuery, + type SnapshotJournalCollectionRequest +} from './SnapshotJournalCollection' + +const epoch = '12345678-1234-4234-9234-123456789012' +const request = (stream: 'scope' | 'physical' = 'scope', limit = 256): SnapshotJournalCollectionRequest => ({ + epoch, + floor: snapshotJournalRevision('50'), + stream, + limit +}) +let k: Knex, directory: string +beforeEach(async () => { + directory = await mkdtemp(join(tmpdir(), 'ts569-collection-')) + k = knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + await k.raw('PRAGMA journal_mode=WAL') + await k.raw('PRAGMA busy_timeout=0') + for (const ddl of [ + SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL, + SNAPSHOT_JOURNAL_SQLITE_GENERATION_DDL, + ...SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL, + ...snapshotJournalReceiptDdl(k) + ]) + await k.raw(ddl) + await k('snapshot_journal_clock').insert({ id: 1, revision: 100, ceiling: 100000, enabled: 1, reason: null }) + await k('snapshot_journal_generation').insert({ + id: 1, + version: 1, + epoch, + source: 'a'.repeat(64), + ceiling: '100000', + complete: 1 + }) + await k('snapshot_journal_retention').insert({ id: 1, floor: '50', receiptLimit: 128, receiptLifetimeMs: 60000 }) +}) +afterEach(async () => { + await k.destroy() + await rm(directory, { recursive: true, force: true }) +}) +const scope = (id1: number, present: number, revision = 10) => ({ + tableId: 0, + userId: 1, + id1, + id2: 0, + exactText: '', + revision, + present +}) + +test('the bound counts live rows; later tombstones require another primary-key page', async () => { + const rows = Array.from({ length: 300 }, (_, i) => scope(i + 1, Number(i < 260), i >= 290 ? 60 : 10)) + for (let n = 0; n < rows.length; n += 100) await k('snapshot_journal_scope').insert(rows.slice(n, n + 100)) + const first = await k.transaction(t => collectSnapshotJournalTombstones(t, request())) + expect(first).toMatchObject({ examined: 256, removed: 0, complete: false }) + expect(await k('snapshot_journal_scope')).toHaveLength(300) + const second = await k.transaction(t => collectSnapshotJournalTombstones(t, { ...request(), after: first!.after })) + expect(second).toMatchObject({ examined: 44, removed: 30, complete: true }) + expect(await k('snapshot_journal_scope').orderBy('id1')).toEqual( + rows.filter(row => row.present === 1 || row.revision > 50) + ) + const sql = snapshotJournalCollectionQuery(k, { ...request(), after: first!.after }).toSQL() + const plan: Array<{ detail: string }> = await k.raw( + 'EXPLAIN QUERY PLAN ' + sql.sql, + sql.bindings as Knex.RawBinding[] + ) + expect(plan.some(row => /SEARCH j USING INDEX sqlite_autoindex_snapshot_journal_scope_1/.test(row.detail))).toBe(true) + expect(plan.some(row => /TEMP B-TREE/.test(row.detail))).toBe(false) + expect(sql.sql).not.toMatch(/where.*(?:present|revision)/) +}) + +test('physical keys preserve exact UTF-8 order and resume after deleted positions', async () => { + const texts = ['😀', 'é', 'Z', 'A'] + const rows = texts.map(exactText => ({ + tableId: 12, + id1: 1, + id2: 0, + exactText, + revision: 40, + generation: 30, + present: 0 + })) + await k('snapshot_journal_physical').insert(rows) + await k('snapshot_journal_scope').insert(scope(1, 0)) + const first = await k.transaction(t => collectSnapshotJournalTombstones(t, request('physical', 2))) + expect(first).toMatchObject({ examined: 2, removed: 2, complete: false, after: { key: { exactText: 'Z' } } }) + const second = await k.transaction(t => + collectSnapshotJournalTombstones(t, { ...request('physical', 2), after: first!.after }) + ) + expect(second).toMatchObject({ removed: 2, after: { key: { exactText: '😀' } } }) + const last = await k.transaction(t => + collectSnapshotJournalTombstones(t, { ...request('physical', 2), after: second!.after }) + ) + expect(last).toEqual({ examined: 0, removed: 0, complete: true, after: second!.after }) + expect(await k('snapshot_journal_scope')).toHaveLength(1) +}) + +test('epoch, stream and floor mismatches never collect another pass', async () => { + await k('snapshot_journal_scope').insert(scope(1, 0)) + const cursor = { ...request(), key: { tableId: 0, userId: 1, id1: 1, id2: 0, exactText: '' } } + for (const change of [ + { epoch: '12345678-1234-4234-9234-123456789013' }, + { floor: snapshotJournalRevision('49') }, + { stream: 'physical' as const } + ]) + await expect( + k.transaction(t => collectSnapshotJournalTombstones(t, { ...request(), after: { ...cursor, ...change } })) + ).rejects.toThrow() + await expect( + k.transaction(t => collectSnapshotJournalTombstones(t, { ...request(), floor: snapshotJournalRevision('49') })) + ).rejects.toThrow() + await k('snapshot_journal_generation').update({ complete: 0 }) + await expect(k.transaction(t => collectSnapshotJournalTombstones(t, request()))).rejects.toThrow() + expect(await k('snapshot_journal_scope')).toHaveLength(1) + expect((await k('snapshot_journal_clock').first()).revision).toBe(100) +}) + +test('collection rollback restores deleted metadata; a lost acknowledgement leaves its committed result', async () => { + await k('snapshot_journal_scope').insert([scope(1, 0), scope(2, 1)]) + await expect( + k.transaction(async t => { + await collectSnapshotJournalTombstones(t, request()) + throw new Error('before commit') + }) + ).rejects.toThrow('before commit') + expect(await k('snapshot_journal_scope')).toHaveLength(2) + await expect( + (async () => { + await k.transaction(t => collectSnapshotJournalTombstones(t, request())) + throw new Error('lost acknowledgement') + })() + ).rejects.toThrow('lost acknowledgement') + expect(await k('snapshot_journal_scope').select('id1')).toEqual([{ id1: 2 }]) +}) + +test.each([ + { present: 2 }, + { revision: 0 }, + { exactText: 'x'.repeat(401) }, + { exactText: Buffer.from([255]) }, + { id1: 0 } +])('corrupt metadata %o refuses the whole page before deletion', async change => { + await k('snapshot_journal_scope').insert([scope(1, 0), { ...scope(2, 0), ...change }]) + await expect(k.transaction(t => collectSnapshotJournalTombstones(t, request()))).rejects.toThrow() + expect(await k('snapshot_journal_scope')).toHaveLength(2) +}) + +test.each([0, 257, Number.MAX_SAFE_INTEGER + 1, '1', null])( + 'rejects invalid bound %p before mutating the clock', + async limit => { + await expect( + k.transaction(t => + collectSnapshotJournalTombstones(t, { ...request(), limit } as SnapshotJournalCollectionRequest) + ) + ).rejects.toThrow() + expect((await k('snapshot_journal_clock').first()).revision).toBe(100) + } +) + +test('a disabled generation never deletes metadata or advances its floor', async () => { + await k('snapshot_journal_scope').insert(scope(1, 0)) + await k('snapshot_journal_clock').update({ enabled: 0, reason: 'capacity-exhausted' }) + expect(await k.transaction(t => collectSnapshotJournalTombstones(t, request()))).toBeUndefined() + expect(await k('snapshot_journal_scope')).toHaveLength(1) + expect((await k('snapshot_journal_retention').first()).floor).toBe('50') +}) + +test('collection requires a caller-owned transaction before allocating a revision', async () => { + await expect(collectSnapshotJournalTombstones(k, request())).rejects.toThrow( + 'Invalid snapshot journal tombstone collection state' + ) + expect((await k('snapshot_journal_clock').first()).revision).toBe(100) +}) + +test('300 seeded independent ledgers preserve the live view across bounded passes', async () => { + let seed = 3242026 + const random = (bound: number) => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0 + return seed % bound + } + for (let n = 0; n < 300; n++) { + const stream = n % 2 ? 'scope' : 'physical', + table = 'snapshot_journal_' + stream + const fields = + stream === 'scope' ? ['tableId', 'userId', 'id1', 'id2', 'exactText'] : ['tableId', 'id1', 'id2', 'exactText'] + const order = (a: Record, b: Record) => { + for (const field of fields) { + const difference = + field === 'exactText' + ? Buffer.compare(Buffer.from(a[field] as string), Buffer.from(b[field] as string)) + : (a[field] as number) - (b[field] as number) + if (difference) return difference + } + return 0 + } + const source = Array.from({ length: 16 }, (_unused, i) => { + const revision = [1, 49, 50, 51, 70][random(5)] + return { + tableId: random(13), + ...(stream === 'scope' ? { userId: random(3) + 1 } : {}), + id1: i + 1, + id2: random(4), + exactText: ['', 'A', 'Z', 'é', '😀', 'é'.repeat(200)][random(6)], + revision, + present: random(2), + ...(stream === 'physical' ? { generation: revision } : {}) + } + }).sort(order) + await k(table).delete() + await k(table).insert(source) + const bound = random(8) + 1 + let input = request(stream, bound), + examined = 0 + while (true) { + const position = input.after?.key + const expected = source.filter(row => position === undefined || order(row, { ...position }) > 0).slice(0, bound) + const result = await k.transaction(t => collectSnapshotJournalTombstones(t, input)) + expect(result).toMatchObject({ + examined: expected.length, + removed: expected.filter(row => row.present === 0 && row.revision <= 50).length, + complete: expected.length < bound + }) + examined += result!.examined + if (expected.length) { + const last = expected[expected.length - 1] + expect(result!.after!.key).toEqual( + Object.fromEntries(fields.map(field => [field, last[field as keyof typeof last]])) + ) + } + if (result!.complete) break + input = { ...input, after: result!.after } + } + expect(examined).toBe(source.length) + const retained = await k(table).select() + retained.sort(order) + expect(retained).toEqual(source.filter(row => row.present === 1 || row.revision > 50)) + } +}, 60000) + +test.each(['mysql', 'mysql2'])( + 'MySQL %s generates a bounded exact composite range with current locks', + async client => { + const mysql = knex({ client: 'mysql2' }) + mysql.client.config.client = client + try { + for (const stream of ['scope', 'physical'] as const) { + const key = { tableId: 12, ...(stream === 'scope' ? { userId: 41 } : {}), id1: 987, id2: 12, exactText: 'é' } + const input = { ...request(stream, 19), after: { epoch, floor: request().floor, stream, key } } + const sql = snapshotJournalCollectionQuery(mysql, input).toSQL() + const fields = + stream === 'scope' ? ['tableId', 'userId', 'id1', 'id2', 'exactText'] : ['tableId', 'id1', 'id2', 'exactText'] + const operands = stream === 'scope' ? [12, 41, 987, 12, Buffer.from('é')] : [12, 987, 12, Buffer.from('é')] + const ranges = fields.map( + (field, i) => + '(' + + [...fields.slice(0, i).map(prefix => '`j`.`' + prefix + '` = ?'), '`j`.`' + field + '` > ?'].join(' and ') + + ')' + ) + const where = '`j`.`tableId` >= ? and (' + ranges.join(' or ') + ')' + expect(sql.sql).toContain('FORCE INDEX (`PRIMARY`)') + expect(sql.sql).toContain( + 'where ' + + where + + ' order by ' + + fields.map(field => '`j`.`' + field + '` asc').join(', ') + + ' limit ? for update nowait' + ) + expect(sql.bindings).toEqual([12, ...fields.flatMap((_field, i) => operands.slice(0, i + 1)), 19]) + expect(sql.sql).toContain('substr(`j`.`exactText`,1,401)') + expect(sql.sql).toContain('substr(CAST(`j`.`revision` AS CHAR),1,20)') + if (stream === 'physical') expect(sql.sql).toContain('substr(CAST(`j`.`generation` AS CHAR),1,20)') + expect(sql.sql).not.toMatch(/where.*(?:present|revision)/) + } + } finally { + await mysql.destroy() + } + } +) + +test('unsupported drivers refuse before a collection query can be constructed', async () => { + for (const client of ['pg', 'mysql-compatible', '', undefined]) { + const unsupported = { client: { config: { client } } } as unknown as Knex + expect(() => snapshotJournalCollectionQuery(unsupported, request())).toThrow(WERR_INVALID_OPERATION) + } +}) + +test('all cursor envelopes and key bounds reject with the established error identity', () => { + const validKey = { tableId: 0, userId: 1, id1: 1, id2: 0, exactText: '' } + const validCursor = { epoch, floor: request().floor, stream: 'scope' as const, key: validKey } + const invalidRequests: unknown[] = [ + undefined, + null, + 1, + 'scope', + Object.assign([], request()), + { ...request(), epoch: null }, + { ...request(), epoch: 'x' + epoch }, + { ...request(), epoch: epoch + 'x' }, + { ...request(), epoch: epoch.replace('-4', '-3') }, + { ...request(), stream: 'unknown' }, + { ...request(), after: null }, + { ...request(), after: Object.assign([], validCursor) }, + { ...request(), after: { ...validCursor, key: null } }, + { ...request(), after: { ...validCursor, key: Object.assign([], validKey) } } + ] + for (const change of [ + { tableId: -1 }, + { tableId: 13 }, + { tableId: 0.5 }, + { userId: 0 }, + { userId: 1.5 }, + { userId: Number.MAX_SAFE_INTEGER + 1 }, + { id1: 0 }, + { id1: 1.5 }, + { id1: '1' }, + { id2: -1 }, + { id2: 0.5 }, + { exactText: null }, + { exactText: '\ud800' }, + { exactText: 'é'.repeat(201) } + ]) + invalidRequests.push({ ...request(), after: { ...validCursor, key: { ...validKey, ...change } } }) + invalidRequests.push({ ...request('physical'), after: { ...validCursor, stream: 'physical', key: validKey } }) + for (const input of invalidRequests) + expect(() => snapshotJournalCollectionQuery(k, input as SnapshotJournalCollectionRequest)).toThrow( + WERR_INVALID_OPERATION + ) + expect(() => + snapshotJournalCollectionQuery(k, { + ...request(), + after: { ...validCursor, key: { ...validKey, exactText: 'é'.repeat(200) } } + }) + ).not.toThrow() +}) + +test.each(['scope', 'physical'] as const)( + 'MySQL %s parses exact stored keys and deletes only the eligible row', + async stream => { + const mysql = knex({ client: 'mysql2' }), + queries: Array<{ sql: string; bindings: unknown[] }> = [] + const row = (exactText: string, revisionText: string, present: number | boolean) => ({ + tableId: 12, + ...(stream === 'scope' ? { userId: '41' } : {}), + id1: '987', + id2: '12', + exactBytes: Buffer.from(exactText), + revisionText, + present, + ...(stream === 'physical' ? { generationText: '30' } : {}) + }) + let records: Array> = [row('Z', '49', false), row('é', '51', false), row('😀', '49', true)] + let generations: Array> = [{ id: 1, version: 1, complete: 1, epoch }] + let floor = '50', + allocated = '9007199254740993', + deleted = 1 + function response(sql: string): unknown { + if (sql.includes('from `snapshot_journal_clock`')) return [{ ceiling: '9223372036854775807' }] + if (sql.includes('from `snapshot_journal_invalid`')) return [] + if (sql.startsWith('SELECT CAST(LAST_INSERT_ID()')) return [{ revision: allocated }] + if (sql.includes('from `snapshot_journal_generation`')) return generations + if (sql.includes('from `snapshot_journal_retention`')) + return [{ id: 1, floor, receiptLimit: 128, receiptLifetimeMs: '60000' }] + if (sql.includes('FORCE INDEX (`PRIMARY`)')) return records + if (sql.startsWith('delete from')) return { affectedRows: deleted } + if (sql.startsWith('insert into')) return { affectedRows: 1, insertId: 1 } + if (/^(BEGIN|COMMIT|ROLLBACK);?$/.test(sql)) return { affectedRows: 0 } + throw new Error('Unexpected synthetic MySQL query: ' + sql) + } + const connection = { + query( + q: { sql: string }, + bindings: unknown[], + callback: (error: Error | null, rows?: unknown, fields?: unknown[]) => void + ) { + queries.push({ sql: q.sql, bindings }) + callback(null, response(q.sql), []) + } + } + jest.spyOn(mysql.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(mysql.client, 'releaseConnection').mockResolvedValue(undefined) + const key = { tableId: 12, ...(stream === 'scope' ? { userId: 41 } : {}), id1: 987, id2: 12, exactText: 'A' }, + input = { ...request(stream, 4), after: { epoch, floor: request().floor, stream, key } } + try { + expect(await mysql.transaction(t => collectSnapshotJournalTombstones(t, input))).toEqual({ + examined: 3, + removed: 1, + complete: true, + after: { epoch, floor: request().floor, stream, key: { ...key, exactText: '😀' } } + }) + const deletes = queries.filter(q => q.sql.startsWith('delete from `snapshot_journal_' + stream + '`')) + expect(deletes).toHaveLength(1) + expect(deletes[0].bindings).toEqual([0, '49', 12, ...(stream === 'scope' ? [41] : []), 987, 12, Buffer.from('Z')]) + expect(deletes[0].sql).toContain('`revision` = CAST(? AS SIGNED)') + for (const table of ['clock', 'generation', 'retention']) + expect(queries.find(q => q.sql.includes('from `snapshot_journal_' + table + '`'))!.sql).toContain( + 'for update nowait' + ) + const gen = queries.find(q => q.sql.includes('from `snapshot_journal_generation`'))! + expect(gen.bindings).toEqual([2]) + expect(gen.sql).toContain('substr(`epoch`,1,37)') + const projection = ['tableId', ...(stream === 'scope' ? ['userId'] : []), 'id1', 'id2', 'present'] + expect( + queries + .find(q => q.sql.includes('FORCE INDEX (`PRIMARY`)'))! + .sql.startsWith('select ' + projection.map(field => '`j`.`' + field + '`').join(', ') + ', substr(') + ).toBe(true) + for (const invalid of [ + ' 987', + '987 ', + '0987', + '+987', + '987.0', + '9.87e2', + '0x3db', + 'Infinity', + '9007199254740992' + ]) { + queries.length = 0 + records = [{ ...row('Z', '49', 0), id1: invalid }] + await expect(mysql.transaction(t => collectSnapshotJournalTombstones(t, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + expect(queries.some(q => q.sql.startsWith('delete from `snapshot_journal_' + stream + '`'))).toBe(false) + } + const refuses = async () => { + queries.length = 0 + await expect(mysql.transaction(t => collectSnapshotJournalTombstones(t, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + expect(queries.some(q => q.sql.startsWith('delete from `snapshot_journal_' + stream + '`'))).toBe(false) + } + const validGeneration = { id: 1, version: 1, complete: 1, epoch } + for (const state of [ + [], + [validGeneration, validGeneration], + [{ ...validGeneration, id: 2 }], + [{ ...validGeneration, version: 2 }], + [{ ...validGeneration, complete: 0 }], + [{ ...validGeneration, epoch: epoch.slice(0, -1) + '3' }] + ]) { + generations = state + records = [row('Z', '50', 0)] + await refuses() + } + generations = [validGeneration] + for (const invalidFloor of ['49', '9007199254740994']) { + floor = invalidFloor + await refuses() + } + floor = '50' + allocated = '49' + await refuses() + allocated = '50' + records = [{ ...row('Z', '50', 0), ...(stream === 'physical' ? { generationText: '50' } : {}) }] + expect(await mysql.transaction(t => collectSnapshotJournalTombstones(t, input))).toMatchObject({ removed: 1 }) + allocated = '9007199254740993' + for (const change of [ + { tableId: '012' }, + { id2: '012' }, + ...(stream === 'scope' ? [{ userId: '041' }] : []), + { exactBytes: Uint8Array.from([255]) }, + { exactBytes: Buffer.alloc(401, 65) }, + { present: '0' }, + { present: 2 }, + { revisionText: '0' }, + ...(stream === 'physical' ? [{ generationText: '0' }, { generationText: '51' }] : []) + ]) { + records = [row('B', '49', 0), { ...row('Z', '50', 0), ...change }] + await refuses() + } + for (const badPage of [ + [row('B', '49', 0), row('B', '49', 0)], + [row('Z', '49', 0), row('B', '49', 0)], + [row('A', '49', 0)], + Array.from({ length: 5 }, (_v, i) => row('B' + i, '49', 0)), + ...['tableId', ...(stream === 'scope' ? ['userId'] : []), 'id1', 'id2'].map(field => [ + row('B', '49', 0), + { ...row('Z', '49', 0), [field]: field === 'tableId' ? 11 : 1 } + ]) + ]) { + records = badPage + await refuses() + } + records = [row('é'.repeat(200), '50', 0)] + expect(await mysql.transaction(t => collectSnapshotJournalTombstones(t, input))).toMatchObject({ removed: 1 }) + for (const affected of [0, 2]) { + deleted = affected + await expect(mysql.transaction(t => collectSnapshotJournalTombstones(t, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + } + deleted = 1 + records = [] + expect(await mysql.transaction(t => collectSnapshotJournalTombstones(t, request(stream)))).toEqual({ + examined: 0, + removed: 0, + complete: true + }) + } finally { + await mysql.destroy() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCollection.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCollection.ts new file mode 100644 index 000000000..e82dceeaa --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalCollection.ts @@ -0,0 +1,292 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { lockSnapshotJournalRetention } from './SnapshotJournalReceipt' +import { reserveSnapshotJournalCaptureFence } from './SnapshotJournalCaptureFence' +import { + compareSnapshotJournalRevisions, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' +import { snapshotJournalRevisionOperand } from './SnapshotJournalRevisionSql' + +export interface SnapshotJournalCollectionKey { + tableId: number + userId?: number + id1: number + id2: number + exactText: string +} +export interface SnapshotJournalCollectionCursor { + epoch: string + floor: SnapshotJournalRevision + stream: 'scope' | 'physical' + key: SnapshotJournalCollectionKey +} +export interface SnapshotJournalCollectionRequest { + epoch: string + floor: SnapshotJournalRevision + stream: 'scope' | 'physical' + after?: SnapshotJournalCollectionCursor + limit: number +} +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid snapshot journal tombstone collection state') +} +function object(value: unknown): value is object { + return value !== null && typeof value === 'object' && !Array.isArray(value) +} +function integer(value: unknown, minimum: number, maximum = Number.MAX_SAFE_INTEGER, stored = false): number { + if (stored && typeof value === 'string' && /^(0|[1-9]\d{0,15})$/.test(value)) value = Number(value) + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum || value > maximum) return invalid() + return value +} +function text(value: unknown): string { + const binary = Buffer.isBuffer(value) || value instanceof Uint8Array ? value : undefined + const result = binary === undefined ? value : Buffer.from(binary).toString('utf8') + if (typeof result !== 'string') return invalid() + const bytes = Buffer.from(result, 'utf8') + if (bytes.length > 400 || bytes.toString('utf8') !== result) return invalid() + if (binary !== undefined && !bytes.equals(Buffer.from(binary))) return invalid() + return result +} +function key(value: SnapshotJournalCollectionKey, stream: 'scope' | 'physical'): SnapshotJournalCollectionKey { + if (!object(value)) return invalid() + const result: SnapshotJournalCollectionKey = { + tableId: integer(value.tableId, 0, 12), + id1: integer(value.id1, 1), + id2: integer(value.id2, 0), + exactText: text(value.exactText) + } + if (stream === 'scope') result.userId = integer(value.userId, 1) + else if (value.userId !== undefined) return invalid() + return result +} +export function snapshotJournalCollectionRequest( + value: SnapshotJournalCollectionRequest +): SnapshotJournalCollectionRequest { + if (!object(value)) return invalid() + const { epoch, stream } = value + if (typeof epoch !== 'string' || !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(epoch)) + return invalid() + if (stream !== 'scope' && stream !== 'physical') return invalid() + const result: SnapshotJournalCollectionRequest = { + epoch, + stream, + floor: snapshotJournalRevision(value.floor), + limit: integer(value.limit, 1, 256) + } + const after = value.after + if (after !== undefined) { + if (!object(after) || after.epoch !== epoch || after.floor !== result.floor || after.stream !== stream) + return invalid() + result.after = { + epoch, + stream, + floor: result.floor, + key: key(after.key, stream) + } + } + return result +} +function local(k: Knex): boolean { + if (['better-sqlite3', 'sqlite3'].includes(k.client.config.client)) return true + if (['mysql2', 'mysql'].includes(k.client.config.client)) return false + return invalid() +} +const columns = (stream: 'scope' | 'physical') => + stream === 'scope' + ? (['tableId', 'userId', 'id1', 'id2', 'exactText'] as const) + : (['tableId', 'id1', 'id2', 'exactText'] as const) +function values( + position: SnapshotJournalCollectionKey, + stream: 'scope' | 'physical', + sqlite: boolean +): Array { + return columns(stream).map(field => + field === 'exactText' && !sqlite ? Buffer.from(position.exactText, 'utf8') : position[field]! + ) +} +function seek(query: Knex.QueryBuilder, k: Knex, request: SnapshotJournalCollectionRequest): void { + if (request.after === undefined) return + const fields = columns(request.stream).map(field => 'j.' + field), + operands = values(request.after.key, request.stream, local(k)) + if (local(k)) { + query.whereRaw('(' + fields.map(() => '??').join(',') + ') > (' + fields.map(() => '?').join(',') + ')', [ + ...fields, + ...operands + ]) + return + } + // Explicit equality-prefix ranges retain every component of MySQL's primary + // index range; do not filter by present/revision before this bounded scan. + query.where(fields[0], '>=', operands[0]).where(function () { + for (const [index, field] of fields.entries()) { + this.orWhere(function () { + for (let prefix = 0; prefix < index; prefix++) this.where(fields[prefix], operands[prefix]) + this.where(field, '>', operands[index]) + }) + } + }) +} +/** Primary-key scan bounds examined metadata, including live and newer rows. */ +export function snapshotJournalCollectionQuery(k: Knex, input: SnapshotJournalCollectionRequest): Knex.QueryBuilder { + const request = snapshotJournalCollectionRequest(input), + sqlite = local(k), + table = 'snapshot_journal_' + request.stream + const hint = sqlite ? '?? AS ?? INDEXED BY ??' : '?? AS ?? FORCE INDEX (??)' + const query = k + .from(k.raw(hint, [table, 'j', sqlite ? 'sqlite_autoindex_' + table + '_1' : 'PRIMARY'])) + .select( + ...columns(request.stream) + .filter(field => field !== 'exactText') + .map(field => 'j.' + field), + 'j.present' + ) + .select( + k.raw(sqlite ? "coalesce(substr(CAST(?? AS BLOB),1,401),'') AS ??" : 'substr(??,1,401) AS ??', [ + 'j.exactText', + 'exactBytes' + ]) + ) + .select( + k.raw(sqlite ? 'substr(CAST(?? AS TEXT),1,20) AS ??' : 'substr(CAST(?? AS CHAR),1,20) AS ??', [ + 'j.revision', + 'revisionText' + ]) + ) + .orderBy(columns(request.stream).map(field => 'j.' + field)) + .limit(request.limit) + if (sqlite) query.select(k.raw('typeof(??) AS ??', ['j.exactText', 'keyType'])) + else query.forUpdate().noWait() + if (request.stream === 'physical') + query.select( + k.raw(sqlite ? 'substr(CAST(?? AS TEXT),1,20) AS ??' : 'substr(CAST(?? AS CHAR),1,20) AS ??', [ + 'j.generation', + 'generationText' + ]) + ) + seek(query, k, request) + return query +} +interface Metadata { + key: SnapshotJournalCollectionKey + revision: SnapshotJournalRevision + present: boolean +} +function metadata(row: Record, stream: 'scope' | 'physical', sqlite: boolean): Metadata { + if (sqlite && row.keyType !== 'text') return invalid() + if (![0, 1, false, true].includes(row.present as number | boolean)) return invalid() + const position = key( + { + tableId: integer(row.tableId, 0, 12, true), + userId: stream === 'scope' ? integer(row.userId, 1, Number.MAX_SAFE_INTEGER, true) : undefined, + id1: integer(row.id1, 1, Number.MAX_SAFE_INTEGER, true), + id2: integer(row.id2, 0, Number.MAX_SAFE_INTEGER, true), + exactText: text(row.exactBytes) + }, + stream + ) + const revision = snapshotJournalRevision(row.revisionText) + if (revision === '0') return invalid() + if (stream === 'physical') { + const generation = snapshotJournalRevision(row.generationText) + if (generation === '0' || compareSnapshotJournalRevisions(generation, revision) > 0) return invalid() + } + return { + key: position, + revision, + present: row.present === 1 || row.present === true + } +} +async function generation(k: Knex, epoch: string): Promise { + const query = k('snapshot_journal_generation') + .select('id', 'version', 'complete', k.raw('substr(??,1,37) AS ??', ['epoch', 'epoch'])) + .limit(2) + if (!local(k)) query.forUpdate().noWait() + const rows = await query + if ( + rows.length !== 1 || + rows[0].id !== 1 || + rows[0].version !== 1 || + rows[0].complete !== 1 || + rows[0].epoch !== epoch + ) + return invalid() +} +function compare( + a: SnapshotJournalCollectionKey, + b: SnapshotJournalCollectionKey, + stream: 'scope' | 'physical' +): number { + for (const field of columns(stream)) { + const order = + field === 'exactText' + ? Buffer.compare(Buffer.from(a.exactText, 'utf8'), Buffer.from(b.exactText, 'utf8')) + : a[field]! - b[field]! + if (order !== 0) return order + } + return 0 +} +/** Internal primitive: caller validates complete owned DDL/source and excludes + * migration before starting this fresh short transaction. It never deletes + * source rows. A cursor binds one epoch, floor and stream; a later floor must + * start a new pass. Complete means end of this bounded primary-key pass, not + * physical file shrink or collection of events after that floor. + */ +export async function collectSnapshotJournalTombstones( + k: Knex, + input: SnapshotJournalCollectionRequest +): Promise< + | { + examined: number + removed: number + complete: boolean + after?: SnapshotJournalCollectionCursor + } + | undefined +> { + const request = snapshotJournalCollectionRequest(input) + if (!k.isTransaction) return invalid() + const highWater = await reserveSnapshotJournalCaptureFence(k) + if (highWater === undefined) return undefined + await generation(k, request.epoch) + const retention = await lockSnapshotJournalRetention(k) + if (retention.floor !== request.floor || compareSnapshotJournalRevisions(retention.floor, highWater) > 0) + return invalid() + const records: Array> = await snapshotJournalCollectionQuery(k, request) + if (records.length > request.limit) return invalid() + const rows = records.map(row => metadata(row, request.stream, local(k))) + let previous = request.after?.key + for (const row of rows) { + if (previous !== undefined && compare(previous, row.key, request.stream) >= 0) return invalid() + previous = row.key + } + let removed = 0 + await runInSeries(rows, async row => { + if (row.present || compareSnapshotJournalRevisions(row.revision, request.floor) > 0) return + const fields = columns(request.stream), + operands = values(row.key, request.stream, local(k)) + const query = k('snapshot_journal_' + request.stream) + .where('present', 0) + .where('revision', snapshotJournalRevisionOperand(k, row.revision)) + for (const [index, field] of fields.entries()) query.where(field, operands[index]) + if ((await query.delete()) !== 1) return invalid() + removed++ + }) + return { + examined: rows.length, + removed, + complete: rows.length < request.limit, + ...(previous === undefined + ? {} + : { + after: { + epoch: request.epoch, + stream: request.stream, + floor: request.floor, + key: previous + } + }) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalConnections.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalConnections.test.ts new file mode 100644 index 000000000..395a8503d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalConnections.test.ts @@ -0,0 +1,439 @@ +import { knex, type Knex } from 'knex' +import { withSnapshotJournalConnections, SnapshotJournalConnectionCleanupError } from './SnapshotJournalConnections' + +function deferred() { + let resolve!: (value: T) => void + let reject!: (reason: unknown) => void + const promise = new Promise((yes, no) => { + resolve = yes + reject = no + }) + return { promise, resolve, reject } +} +function pool() { + return knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) +} +async function fixture() { + const writer = pool(), + reader = pool() + return { + writer, + reader, + close: async () => { + await writer.destroy() + await reader.destroy() + } + } +} + +test('both single-slot pools are reserved before work; release permits later independent work', async () => { + const f = await fixture() + try { + const started = deferred(), + finish = deferred() + const operation = withSnapshotJournalConnections( + f.writer, + f.reader, + () => undefined, + async (a, b) => { + expect(a).not.toBe(b) + expect(f.writer.client.pool.numUsed()).toBe(1) + expect(f.reader.client.pool.numUsed()).toBe(1) + await f.writer.raw('CREATE TABLE x (id INTEGER)').connection(a) + await f.reader.raw('CREATE TABLE y (id INTEGER)').connection(b) + started.resolve() + await finish.promise + return 7 + } + ) + await started.promise + expect(f.writer.client.pool.numUsed()).toBe(1) + finish.resolve() + expect(await operation).toBe(7) + expect(f.writer.client.pool.numUsed()).toBe(0) + expect(f.reader.client.pool.numUsed()).toBe(0) + await f.writer('x').insert({ id: 1 }) + await f.reader('y').insert({ id: 2 }) + } finally { + await f.close() + } +}) + +test.each(['writer', 'reader'] as const)('an occupied %s pool cannot run the barrier callback', async held => { + const f = await fixture() + const hold = f[held], + connection = await hold.client.acquireConnection() + try { + const other = held === 'writer' ? f.reader : f.writer + const acquired = deferred() + const acquire = other.client.acquireConnection.bind(other.client) + jest.spyOn(other.client, 'acquireConnection').mockImplementation(async () => { + const value = await acquire() + acquired.resolve() + return value + }) + let cancelled = false + const action = jest.fn(async () => 7) + const operation = withSnapshotJournalConnections( + f.writer, + f.reader, + () => { + if (cancelled) throw new Error('cancelled') + }, + action + ) + const rejection = expect(operation).rejects.toThrow('cancelled') + await acquired.promise + expect(action).not.toHaveBeenCalled() + cancelled = true + await hold.client.releaseConnection(connection) + await rejection + expect(action).not.toHaveBeenCalled() + expect(other.client.pool.numUsed()).toBe(0) + expect(hold.client.pool.numUsed()).toBe(0) + } finally { + await f.close() + } +}) + +test.each(['writer', 'reader'] as const)( + 'failed %s acquisition drains a late successful peer before rejecting', + async failed => { + const f = await fixture() + try { + const late = failed === 'writer' ? f.reader : f.writer + const allow = deferred(), + entered = deferred() + const acquire = late.client.acquireConnection.bind(late.client) + jest.spyOn(late.client, 'acquireConnection').mockImplementation(async () => { + entered.resolve() + await allow.promise + return await acquire() + }) + const failure = new Error('acquire failed') + jest.spyOn(f[failed].client, 'acquireConnection').mockRejectedValue(failure) + let settled = false + const action = jest.fn(async () => 7) + const operation = withSnapshotJournalConnections(f.writer, f.reader, () => undefined, action) + void operation.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + const rejection = expect(operation).rejects.toThrow('Snapshot connection acquisition failed') + await entered.promise + expect(settled).toBe(false) + expect(action).not.toHaveBeenCalled() + allow.resolve() + await rejection + await expect(operation).rejects.toMatchObject({ errors: [failure] }) + expect(late.client.pool.numUsed()).toBe(0) + expect(action).not.toHaveBeenCalled() + } finally { + await f.close() + } + } +) + +test('run failure still releases both pools and retains the original error', async () => { + const f = await fixture() + try { + const error = new Error('pin failed') + await expect( + withSnapshotJournalConnections( + f.writer, + f.reader, + () => undefined, + async () => { + throw error + } + ) + ).rejects.toBe(error) + expect(f.writer.client.pool.numUsed()).toBe(0) + expect(f.reader.client.pool.numUsed()).toBe(0) + } finally { + await f.close() + } +}) + +test('cleanup failure waits for the other release and preserves both failure causes', async () => { + const f = await fixture() + const owned: unknown[] = [] + const releaseWriter = f.writer.client.releaseConnection.bind(f.writer.client) + const releaseReader = f.reader.client.releaseConnection.bind(f.reader.client) + try { + const release = deferred(), + entered = deferred() + jest.spyOn(f.writer.client, 'releaseConnection').mockRejectedValue(new Error('writer release failed')) + jest.spyOn(f.reader.client, 'releaseConnection').mockImplementation(async value => { + entered.resolve() + await release.promise + await releaseReader(value) + }) + let settled = false + const operation = withSnapshotJournalConnections( + f.writer, + f.reader, + () => undefined, + async (a, b) => { + owned.push(a, b) + throw new Error('read failed') + } + ) + void operation.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + const rejection = expect(operation).rejects.toBeInstanceOf(SnapshotJournalConnectionCleanupError) + await entered.promise + expect(settled).toBe(false) + release.resolve() + await rejection + await expect(operation).rejects.toMatchObject({ + name: 'SnapshotJournalConnectionCleanupError', + message: 'Snapshot journal connection cleanup failed', + cause: { + message: 'Snapshot connection ownership did not drain', + errors: [ + expect.objectContaining({ message: 'read failed' }), + expect.objectContaining({ message: 'writer release failed' }) + ] + } + }) + expect(f.reader.client.pool.numUsed()).toBe(0) + } finally { + jest.restoreAllMocks() + if (owned.length) await releaseWriter(owned[0]) + await f.close() + } +}) + +test('distinct wrappers around the same client refuse before acquisition', async () => { + const f = await fixture() + try { + const alias = { ...f.reader, client: f.writer.client } as Knex + const acquire = jest.spyOn(f.writer.client, 'acquireConnection') + const action = jest.fn(async () => 1) + await expect(withSnapshotJournalConnections(f.writer, alias, () => undefined, action)).rejects.toThrow( + 'Snapshot capture requires two independent owned connection pools' + ) + expect(acquire).not.toHaveBeenCalled() + expect(action).not.toHaveBeenCalled() + } finally { + jest.restoreAllMocks() + await f.close() + } +}) + +test('two pools returning the same native handle release both reservations without running the barrier', async () => { + const f = await fixture() + const connection = await f.writer.client.acquireConnection() + const release = f.writer.client.releaseConnection.bind(f.writer.client) + try { + jest.spyOn(f.writer.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(f.reader.client, 'acquireConnection').mockResolvedValue(connection) + const drain = jest.fn(async () => undefined) + const action = jest.fn(async () => 1) + await expect(withSnapshotJournalConnections(f.writer, f.reader, () => undefined, action, drain)).rejects.toThrow( + 'Snapshot pools returned the same native connection' + ) + expect(action).not.toHaveBeenCalled() + expect(drain.mock.calls).toEqual([ + [f.writer, connection], + [f.reader, connection] + ]) + } finally { + jest.restoreAllMocks() + await release(connection) + await f.close() + } +}) + +test('a successful callback with failed release reports only the release failure', async () => { + const f = await fixture() + const failure = new Error('owned writer release failed') + try { + const operation = withSnapshotJournalConnections( + f.writer, + f.reader, + () => undefined, + async () => 42, + async (owner, connection) => { + await owner.client.releaseConnection(connection) + if (owner === f.writer) throw failure + } + ) + await expect(operation).rejects.toMatchObject({ + name: 'SnapshotJournalConnectionCleanupError', + message: 'Snapshot journal connection cleanup failed', + cause: { message: 'Snapshot connection ownership did not drain', errors: [failure] } + }) + expect(f.writer.client.pool.numUsed()).toBe(0) + expect(f.reader.client.pool.numUsed()).toBe(0) + } finally { + await f.close() + } +}) + +test('already cancelled calls and shared pools refuse before native acquisition', async () => { + const f = await fixture() + try { + const spy = jest.spyOn(f.writer.client, 'acquireConnection') + await expect( + withSnapshotJournalConnections( + f.writer, + f.reader, + () => { + throw new Error('already cancelled') + }, + async () => 1 + ) + ).rejects.toThrow('already cancelled') + await expect( + withSnapshotJournalConnections( + f.writer, + f.writer, + () => undefined, + async () => 1 + ) + ).rejects.toThrow('independent owned') + expect(spy).not.toHaveBeenCalled() + } finally { + await f.close() + } +}) + +test('caller-owned transactions cannot be borrowed as new pool reservations', async () => { + const f = await fixture() + try { + await f.writer.transaction(async trx => { + await expect( + withSnapshotJournalConnections( + trx, + f.reader, + () => undefined, + async () => 1 + ) + ).rejects.toThrow('independent owned') + await expect( + withSnapshotJournalConnections( + f.reader, + trx, + () => undefined, + async () => 1 + ) + ).rejects.toThrow('independent owned') + }) + } finally { + await f.close() + } +}) + +test('generated acquisition schedules preserve admission and release every acquired native connection', async () => { + const fc = (await import('fast-check')).default + await fc.assert( + fc.asyncProperty( + fc.record({ + readerFirst: fc.boolean(), + failWriter: fc.boolean(), + failReader: fc.boolean(), + cancelBefore: fc.boolean(), + cancelWhileAcquiring: fc.boolean(), + failRun: fc.boolean() + }), + async schedule => { + const f = await fixture() + const writerGate = deferred(), + readerGate = deferred() + const writerSeen = deferred(), + readerSeen = deferred() + const acquired: string[] = [], + released: string[] = [] + let cancelled = schedule.cancelBefore + function prepare( + role: 'writer' | 'reader', + gate: ReturnType>, + seen: ReturnType>, + fail: boolean + ) { + const owner = f[role], + acquire = owner.client.acquireConnection.bind(owner.client), + release = owner.client.releaseConnection.bind(owner.client) + jest.spyOn(owner.client, 'acquireConnection').mockImplementation(async () => { + await gate.promise + if (fail) { + seen.resolve() + throw new Error(role + ' unavailable') + } + const connection = await acquire() + acquired.push(role) + seen.resolve() + return connection + }) + jest.spyOn(owner.client, 'releaseConnection').mockImplementation(async connection => { + await release(connection) + released.push(role) + }) + } + try { + prepare('writer', writerGate, writerSeen, schedule.failWriter) + prepare('reader', readerGate, readerSeen, schedule.failReader) + const action = jest.fn(async () => { + if (schedule.failRun) throw new Error('capture failed') + return 'captured' + }) + const operation = withSnapshotJournalConnections( + f.writer, + f.reader, + () => { + if (cancelled) throw new Error('cancelled') + }, + action + ) + const outcome = operation.then( + value => ({ ok: true, value }), + error => ({ ok: false, error }) + ) + if (!schedule.cancelBefore) { + const first = schedule.readerFirst ? [readerGate, readerSeen] : [writerGate, writerSeen] + first[0].resolve() + await first[1].promise + expect(action).not.toHaveBeenCalled() + cancelled = schedule.cancelWhileAcquiring + } + writerGate.resolve() + readerGate.resolve() + const result = await outcome + const admitted = + !schedule.cancelBefore && !schedule.cancelWhileAcquiring && !schedule.failWriter && !schedule.failReader + expect(action).toHaveBeenCalledTimes(Number(admitted)) + expect(result.ok).toBe(admitted && !schedule.failRun) + expect(new Set(released)).toEqual(new Set(acquired)) + expect(new Set(released).size).toBe(released.length) + expect(f.writer.client.pool.numUsed()).toBe(0) + expect(f.reader.client.pool.numUsed()).toBe(0) + } finally { + jest.restoreAllMocks() + await f.close() + } + } + ), + { + numRuns: 300, + seed: 3242026, + ...(process.env.FAST_CHECK_PATH ? { path: process.env.FAST_CHECK_PATH } : {}) + } + ) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalConnections.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalConnections.ts new file mode 100644 index 000000000..469b6e915 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalConnections.ts @@ -0,0 +1,72 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' + +export class SnapshotJournalConnectionCleanupError extends Error { + constructor(override readonly cause: unknown) { + super('Snapshot journal connection cleanup failed') + this.name = 'SnapshotJournalConnectionCleanupError' + } +} + +interface ReservedConnection { + owner: Knex + connection: unknown +} + +/** Reserve both dedicated pools before any caller can obtain a writer barrier. + * The surrounding retained-view lifecycle controls cancellation and admission: + * assertActive must reject after cancellation/expiry, and its closed promise + * must await this function even when opening has already rejected. These pools + * must be owned by that lifecycle, and destroyed after it settles. This helper + * does not establish matching backend identity, begin a transaction or publish + * a reader. Acquisition failures still drain the other pending acquisition. + */ +export async function withSnapshotJournalConnections( + writer: Knex, + reader: Knex, + assertActive: () => void, + run: (writerConnection: unknown, readerConnection: unknown) => Promise, + release: (owner: Knex, connection: unknown) => Promise = async (owner, connection) => { + await owner.client.releaseConnection(connection) + } +): Promise { + assertActive() + if (writer === reader || writer.client === reader.client || writer.isTransaction || reader.isTransaction) + throw new WERR_INVALID_OPERATION('Snapshot capture requires two independent owned connection pools') + const reserved: ReservedConnection[] = [] + async function reserve(owner: Knex): Promise { + const connection: unknown = await owner.client.acquireConnection() + reserved.push({ owner, connection }) + return connection + } + let result: { ok: true; value: T } | { ok: false; error: unknown } + try { + const [write, read] = await Promise.allSettled([reserve(writer), reserve(reader)]) + if (write.status === 'rejected' || read.status === 'rejected') + throw new AggregateError( + [write, read].filter(value => value.status === 'rejected').map(value => value.reason), + 'Snapshot connection acquisition failed' + ) + if (write.value === read.value) + throw new WERR_INVALID_OPERATION('Snapshot pools returned the same native connection') + assertActive() + result = { ok: true, value: await run(write.value, read.value) } + } catch (error) { + result = { ok: false, error } + } + const cleanup = await Promise.allSettled( + reserved.map(async ({ owner, connection }) => { + await release(owner, connection) + }) + ) + const failedCleanup = cleanup.filter(value => value.status === 'rejected') + if (failedCleanup.length) { + const errors = failedCleanup.map(value => value.reason) + if (!result.ok) errors.unshift(result.error) + throw new SnapshotJournalConnectionCleanupError( + new AggregateError(errors, 'Snapshot connection ownership did not drain') + ) + } + if (!result.ok) throw result.error + return result.value +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalHighWater.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalHighWater.test.ts new file mode 100644 index 000000000..f84d1e638 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalHighWater.test.ts @@ -0,0 +1,223 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { readSnapshotJournalHighWater } from './SnapshotJournalHighWater' +import { snapshotJournalRevision as rev, type SnapshotJournalRevision } from './SnapshotJournalRevision' +import { SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL } from './SnapshotJournalSqliteObservers' + +function open(filename = ':memory:'): Knex { + return knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) +} +async function install(k: Knex) { + for (const sql of SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL) await k.raw(sql) +} + +test('high-water uses fifteen indexed single-row reads, exact values and only the bound profile plus global streams', async () => { + const k = open() + try { + await install(k) + await k('snapshot_journal_scope').insert([ + { + tableId: 0, + userId: 1, + id1: 1, + id2: 0, + exactText: '', + present: 1, + revision: '9007199254740993' + }, + { + tableId: 12, + userId: 1, + id1: 2, + id2: 0, + exactText: 'field', + present: 0, + revision: '9007199254740995' + }, + { + tableId: 0, + userId: 2, + id1: 3, + id2: 0, + exactText: '', + present: 1, + revision: '9223372036854775807' + } + ]) + await k('snapshot_journal_physical').insert([ + { + tableId: 8, + id1: 1, + id2: 0, + exactText: '', + present: 1, + revision: '9007199254740997', + generation: '9007199254740997' + }, + { + tableId: 0, + id1: 3, + id2: 0, + exactText: '', + present: 1, + revision: '9223372036854775807', + generation: '9223372036854775807' + } + ]) + const queries: Array<{ sql: string; bindings: unknown[] }> = [] + const onQuery = (q: { sql: string; bindings: unknown[] }) => { + if (q.sql.startsWith('select')) queries.push(q) + } + k.on('query', onQuery) + expect(await readSnapshotJournalHighWater(k, 1, rev('0'), rev('0'))).toBe('9007199254740997') + k.off('query', onQuery) + expect(queries).toHaveLength(15) + for (const query of queries) { + const plan: Array<{ detail: string }> = await k.raw( + 'EXPLAIN QUERY PLAN ' + query.sql, + query.bindings as Knex.RawBinding[] + ) + expect(plan.some(row => row.detail.includes('SEARCH j USING COVERING INDEX'))).toBe(true) + expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false) + expect(query.bindings.at(-1)).toBe(1) + } + expect(await readSnapshotJournalHighWater(k, 1, rev('9223372036854775807'), rev('9007199254740997'))).toBe( + '9223372036854775807' + ) + } finally { + await k.destroy() + } +}) + +test('empty and quiescent streams retain the exact committed prefix', async () => { + const k = open() + try { + await install(k) + expect(await readSnapshotJournalHighWater(k, 1, rev('9007199254740993'), rev('0'))).toBe('9007199254740993') + } finally { + await k.destroy() + } +}) + +test.each([ + { id: 0, minimum: '0', floor: '0' }, + { id: 1.1, minimum: '0', floor: '0' }, + { id: 1, minimum: '01', floor: '0' }, + { id: 1, minimum: '1', floor: '2' } +])('invalid profile or stale prefix refuses before SQL: %j', async ({ id, minimum, floor }) => { + const k = open() + let queries = 0 + k.on('query', () => queries++) + try { + await expect( + readSnapshotJournalHighWater(k, id, minimum as SnapshotJournalRevision, floor as SnapshotJournalRevision) + ).rejects.toThrow( + id < 1 || !Number.isSafeInteger(id) + ? 'Invalid snapshot journal profile' + : minimum === '01' + ? 'Invalid snapshot journal revision' + : 'Snapshot journal continuity was collected' + ) + expect(queries).toBe(0) + } finally { + await k.destroy() + } +}) + +test('pinned WAL heads remain coherent after an independent writer commits', async () => { + const directory = await mkdtemp(join(tmpdir(), 'ts569-journal-head-')), + filename = join(directory, 'head.sqlite'), + k = open(filename), + writer = open(filename) + let view: Knex.Transaction | undefined + try { + await k.raw('PRAGMA journal_mode=WAL') + await install(k) + await k('snapshot_journal_scope').insert({ + tableId: 3, + userId: 1, + id1: 1, + id2: 0, + exactText: '', + present: 1, + revision: '9007199254740993' + }) + view = await k.transaction() + expect(await readSnapshotJournalHighWater(view, 1, rev('0'), rev('0'))).toBe('9007199254740993') + await writer('snapshot_journal_scope').update({ revision: '9007199254740995' }) + expect(await readSnapshotJournalHighWater(view, 1, rev('0'), rev('0'))).toBe('9007199254740993') + expect(await readSnapshotJournalHighWater(writer, 1, rev('0'), rev('0'))).toBe('9007199254740995') + } finally { + await view?.rollback() + await k.destroy() + await writer.destroy() + await rm(directory, { recursive: true, force: true }) + } +}) + +test.each(['mysql', 'mysql2'])('MySQL high-water alias %s uses exact fifteen bounded index seeks', async client => { + const k = knex({ client: 'mysql2' }) + k.client.config.client = client + const queries: Array<{ sql: string; bindings: unknown[] }> = [] + const connection = { + query( + query: { sql: string }, + bindings: unknown[], + callback: (error: Error | null, rows?: unknown[], fields?: unknown[]) => void + ) { + queries.push({ sql: query.sql, bindings }) + const revisionText = queries.length === 9 ? '9007199254740997' : '9007199254740993' + callback(null, [{ revisionText }], []) + } + } + jest.spyOn(k.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(k.client, 'releaseConnection').mockResolvedValue(undefined) + try { + expect(await readSnapshotJournalHighWater(k, 41, rev('9007199254740992'), rev('0'))).toBe('9007199254740997') + expect(queries).toHaveLength(15) + queries.forEach((query, i) => { + const scope = i < 13, + table = scope ? 'scope' : 'physical', + tableId = scope ? i : i - 5 + expect(query.sql).toBe( + 'select CAST(`j`.`revision` AS CHAR) as `revisionText` from `snapshot_journal_' + + table + + '` AS `j` FORCE INDEX (`snapshot_journal_' + + table + + '_page`) where `j`.`tableId` = ?' + + (scope ? ' and `j`.`userId` = ?' : '') + + ' order by `j`.`revision` desc limit ?' + ) + expect(query.bindings).toEqual(scope ? [tableId, 41, 1] : [tableId, 1]) + }) + } finally { + await k.destroy() + } +}) + +test('SQLite legacy driver alias retains indexed exact high-water reads', async () => { + const k = open() + k.client.config.client = 'sqlite3' + try { + await install(k) + expect(await readSnapshotJournalHighWater(k, 1, rev('9007199254740993'), rev('0'))).toBe('9007199254740993') + } finally { + await k.destroy() + } +}) + +test('unsupported high-water driver refuses before query construction', async () => { + const raw = jest.fn(), + k = { client: { config: { client: 'unsupported' } }, raw } as unknown as Knex + await expect(readSnapshotJournalHighWater(k, 1, rev('0'), rev('0'))).rejects.toThrow( + 'Unsupported snapshot journal SQL driver' + ) + expect(raw).not.toHaveBeenCalled() +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalHighWater.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalHighWater.ts new file mode 100644 index 000000000..128527fe6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalHighWater.ts @@ -0,0 +1,48 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { + compareSnapshotJournalRevisions, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' +import { snapshotJournalRevisionText } from './SnapshotJournalRevisionSql' + +/** Fixed indexed head seeks in the pinned view. MySQL callers retain the exclusive writer barrier during pinning. */ +export async function readSnapshotJournalHighWater( + k: Knex, + userId: number, + minimum: SnapshotJournalRevision, + floor: SnapshotJournalRevision +): Promise { + const client = k.client.config.client, + local = client === 'sqlite3' || client === 'better-sqlite3' + if (!local && client !== 'mysql' && client !== 'mysql2') + throw new WERR_INVALID_OPERATION('Unsupported snapshot journal SQL driver') + if (!Number.isSafeInteger(userId) || userId < 1) throw new WERR_INVALID_OPERATION('Invalid snapshot journal profile') + snapshotJournalRevision(minimum) + snapshotJournalRevision(floor) + if (compareSnapshotJournalRevisions(minimum, floor) < 0) + throw new WERR_INVALID_OPERATION('Snapshot journal continuity was collected') + let high = minimum + const streams = [ + ...Array.from({ length: 13 }, (_, tableId) => ({ stream: 'scope', tableId })), + ...[8, 9].map(tableId => ({ stream: 'physical', tableId })) + ] + await runInSeries(streams, async ({ stream, tableId }) => { + const table = 'snapshot_journal_' + stream + const query = k + .from(k.raw(local ? '?? AS ?? INDEXED BY ??' : '?? AS ?? FORCE INDEX (??)', [table, 'j', table + '_page'])) + .where('j.tableId', tableId) + .select({ revisionText: snapshotJournalRevisionText(k, 'j.revision') }) + .orderBy('j.revision', 'desc') + .first() + if (stream === 'scope') query.where('j.userId', userId) + const row: { revisionText: unknown } | undefined = await query + if (row !== undefined) { + const value = snapshotJournalRevision(row.revisionText) + if (compareSnapshotJournalRevisions(value, high) > 0) high = value + } + }) + return high +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenance.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenance.test.ts new file mode 100644 index 000000000..e05c6403d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenance.test.ts @@ -0,0 +1,501 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { seedArchiveClosure } from '../../../../test/utils/snapshotArchiveFixtures' +import { maintainSnapshotJournal, type SnapshotJournalMaintenanceRequest } from './SnapshotJournalMaintenance' +import * as Maintenance from './SnapshotJournalMaintenance' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' +import * as Sqlite from './SnapshotJournalSqliteGeneration' +import * as Mysql from './SnapshotJournalMysqlGeneration' +import * as Backend from './SnapshotJournalCaptureBackend' +import * as Connections from './SnapshotJournalConnections' +import * as Fence from './SnapshotJournalMaintenanceFence' +import * as Receipts from './SnapshotJournalReceipt' +import { copySnapshotJournalBootstrapPage } from './SnapshotJournalBootstrap' + +const identity = '02' + '11'.repeat(32) +let ceiling: SnapshotJournalRevision +beforeEach(() => { + ceiling = snapshotJournalRevision('1000000') +}) +const receiptPolicy = { receiptLimit: 128, receiptLifetimeMs: 600000 } +const epoch = '00000000-0000-4000-8000-000000000000' +const input = (id = epoch): SnapshotJournalMaintenanceRequest => ({ + epoch: id, + ceiling, + receiptPolicy, + operation: { kind: 'floor', floor: snapshotJournalRevision('0') } +}) +function gate() { + let resolve!: () => void + const promise = new Promise(yes => { + resolve = yes + }) + return { promise, resolve } +} +async function fixture(complete = true) { + const directory = await mkdtemp(join(tmpdir(), 'ts569-maintenance-')) + const k = knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + try { + await storage.migrate('owned maintenance', 'synthetic-maintenance') + await storage.makeAvailable() + await k.raw('PRAGMA journal_mode=WAL') + const { user } = await storage.findOrInsertUser(identity), + { user: other } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(storage, user.userId, other.userId) + await Sqlite.installSnapshotJournalSqliteGeneration(k, ceiling, receiptPolicy) + let state = await Sqlite.readSnapshotJournalSqliteGeneration(k, receiptPolicy) + if (complete) { + let finished = false + for (let n = 0; n < 80 && !finished; n++) finished = (await copySnapshotJournalBootstrapPage(k, 1000000)).complete + expect(finished).toBe(true) + state = await Sqlite.completeSnapshotJournalSqliteGeneration(k, receiptPolicy) + } + return { + k, + storage, + request: input(state.epoch), + async close() { + await storage.destroy() + await rm(directory, { recursive: true, force: true }) + } + } + } catch (error) { + await storage.destroy() + await rm(directory, { recursive: true, force: true }) + throw error + } +} +afterEach(() => jest.restoreAllMocks()) + +test('the provider uses owned WAL transactions for floor and bounded collection', async () => { + const f = await fixture() + try { + const result = await f.storage.maintainSnapshotJournal(f.request) + expect(result).toMatchObject({ kind: 'floor', value: { floor: '0' } }) + await f.storage.awaitSnapshotJournalMaintenanceCleanup() + const page = await f.storage.maintainSnapshotJournal({ + ...f.request, + operation: { + kind: 'collect', + page: { epoch: f.request.epoch, floor: snapshotJournalRevision('0'), stream: 'scope', limit: 2 } + } + }) + expect(page).toMatchObject({ kind: 'collect', value: { examined: 2, removed: 0, complete: false } }) + expect(await f.k('snapshot_journal_receipts')).toHaveLength(0) + } finally { + await f.close() + } +}) +test('invalid maintenance input rejects with its cause and releases admission for a valid request', async () => { + const f = await fixture() + try { + await expect(f.storage.awaitSnapshotJournalMaintenanceCleanup()).resolves.toBeUndefined() + await expect(f.storage.maintainSnapshotJournal({ ...f.request, epoch: 'foreign' })).rejects.toThrow( + 'Snapshot journal maintenance generation is unavailable or changed' + ) + expect(await f.storage.maintainSnapshotJournal(f.request)).toMatchObject({ kind: 'floor', value: { floor: '0' } }) + await f.storage.awaitSnapshotJournalMaintenanceCleanup() + } finally { + await f.close() + } +}) +test('destruction fences pending maintenance configuration before database access', async () => { + const f = await fixture(), + entered = gate(), + release = gate(), + maintain = Maintenance.maintainSnapshotJournal, + acquire = jest.spyOn(f.k.client, 'acquireConnection') + let refused: unknown + jest.spyOn(Maintenance, 'maintainSnapshotJournal').mockImplementation((configure, ...rest) => + maintain( + async () => { + entered.resolve() + await release.promise + try { + return await configure() + } catch (error) { + refused = error + throw error + } + }, + ...rest + ) + ) + try { + const running = f.storage.maintainSnapshotJournal(f.request), + outcome = running.then( + value => ({ value }), + error => ({ error }) + ) + await entered.promise + const destruction = f.storage.destroy().then( + value => ({ value }), + error => ({ error }) + ) + release.resolve() + const destroyed = await destruction + expect(refused).toMatchObject({ message: 'Snapshot journal maintenance is unavailable after destruction begins' }) + expect(destroyed).toHaveProperty('error', refused) + expect(await outcome).toHaveProperty('error') + expect(acquire).not.toHaveBeenCalled() + } finally { + release.resolve() + jest.restoreAllMocks() + await f.close() + } +}) +test('provider destruction stops and drains active maintenance before closing the foreground pool', async () => { + const f = await fixture(), + entered = gate(), + release = gate(), + advance = Receipts.advanceSnapshotJournalFloor + jest.spyOn(Receipts, 'advanceSnapshotJournalFloor').mockImplementation(async (...args) => { + const value = await advance(...args) + entered.resolve() + await release.promise + return value + }) + try { + const running = f.storage.maintainSnapshotJournal(f.request), + outcome = running.then( + value => ({ value }), + error => ({ error }) + ) + await entered.promise + let drained = false + const destruction = f.storage.destroy().then(() => { + drained = true + }) + await new Promise(resolve => setImmediate(resolve)) + expect(drained).toBe(false) + release.resolve() + await destruction + expect(await outcome).toMatchObject({ + error: expect.objectContaining({ message: 'Snapshot journal maintenance is closed' }) + }) + await f.storage.awaitSnapshotJournalMaintenanceCleanup() + expect(drained).toBe(true) + } finally { + release.resolve() + await f.close() + } +}) +test('request binding is validated before native configuration or allocation', () => { + const resolve = jest.fn, []>() + for (const value of [ + { ...input(), epoch: 'bad' }, + { ...input(), epoch: 'prefix' + epoch }, + { ...input(), epoch: epoch + 'suffix' }, + { ...input(), epoch: { toString: () => epoch } }, + { ...input(), ceiling: '0' }, + { ...input(), operation: { kind: 'bad' } }, + { + ...input(), + operation: { + kind: 'bad', + page: { epoch, floor: '0', stream: 'scope', limit: 2 } + } + }, + { + ...input(), + operation: { + kind: 'collect', + page: { epoch: '10000000-0000-4000-8000-000000000000', floor: '0', stream: 'scope', limit: 2 } + } + } + ]) + expect(() => maintainSnapshotJournal(resolve, value as SnapshotJournalMaintenanceRequest)).toThrow() + expect(resolve).not.toHaveBeenCalled() +}) +test('request fields are detached before asynchronous configuration work', async () => { + const f = await fixture(), + entered = gate(), + release = gate() + try { + const original = f.request + const task = maintainSnapshotJournal(async () => { + entered.resolve() + await release.promise + return f.k.client.config + }, original) + await entered.promise + original.epoch = epoch + original.receiptPolicy.receiptLimit = 1 + original.operation = { kind: 'floor', floor: snapshotJournalRevision('999999') } + release.resolve() + expect(await task.result).toMatchObject({ kind: 'floor', value: { floor: '0' } }) + await task.closed + } finally { + release.resolve() + receiptPolicy.receiptLimit = 128 + await f.close() + } +}) +test('unsupported configuration refuses before creating a native owner', async () => { + const task = maintainSnapshotJournal(async () => undefined, input()) + await expect(task.result).rejects.toThrow('requires file-backed') + await expect(task.closed).rejects.toThrow('requires file-backed') +}) +test.each(['incomplete', 'stale epoch', 'wrong ceiling', 'changed source DDL'] as const)( + 'refuses %s without advancing retention', + async state => { + const f = await fixture(state !== 'incomplete') + try { + if (state === 'stale epoch') f.request.epoch = epoch + if (state === 'wrong ceiling') f.request.ceiling = snapshotJournalRevision('999999') + if (state === 'changed source DDL') + await f.k.schema.alterTable('tx_labels', table => table.integer('unowned_column')) + await expect(f.storage.maintainSnapshotJournal(f.request)).rejects.toThrow() + await expect(f.storage.awaitSnapshotJournalMaintenanceCleanup()).resolves.toBeUndefined() + expect((await f.k('snapshot_journal_retention').first()).floor).toBe('0') + } finally { + await f.close() + } + } +) +test('disabled generation commits no floor or metadata deletion', async () => { + const f = await fixture() + try { + await f.k('snapshot_journal_clock').update({ enabled: 0, reason: 'capacity-exhausted' }) + expect(await f.storage.maintainSnapshotJournal(f.request)).toEqual({ kind: 'floor', value: undefined }) + expect((await f.k('snapshot_journal_retention').first()).floor).toBe('0') + } finally { + await f.close() + } +}) +test('cancellation retains source admission through rollback and physical cleanup', async () => { + const f = await fixture(), + entered = gate(), + release = gate(), + abort = new AbortController() + const advance = Receipts.advanceSnapshotJournalFloor + jest.spyOn(Receipts, 'advanceSnapshotJournalFloor').mockImplementation(async (...args) => { + const value = await advance(...args) + entered.resolve() + await release.promise + return value + }) + try { + const opening = f.storage.maintainSnapshotJournal(f.request, { signal: abort.signal }), + rejection = expect(opening).rejects.toThrow('cancelled') + await entered.promise + abort.abort() + await rejection + await expect(f.storage.maintainSnapshotJournal(f.request)).rejects.toThrow('already') + await expect(f.storage.openSnapshotJournalSource(identity, { ceiling, receiptPolicy })).rejects.toThrow('already') + let drained = false + const cleanup = f.storage.awaitSnapshotJournalMaintenanceCleanup().then(() => { + drained = true + }) + await new Promise(resolve => setImmediate(resolve)) + expect(drained).toBe(false) + release.resolve() + await cleanup + expect((await f.k('snapshot_journal_retention').first()).floor).toBe('0') + } finally { + release.resolve() + await f.close() + } +}) +test('physical cleanup failure remains observable and permanently fences admission', async () => { + const f = await fixture(), + close = Backend.closeSnapshotJournalCapturePool, + failure = new Error('synthetic pool closure failure') + jest.spyOn(Backend, 'closeSnapshotJournalCapturePool').mockImplementation(async (...args) => { + await close(...args) + throw failure + }) + try { + await expect(f.storage.maintainSnapshotJournal(f.request)).rejects.toBeInstanceOf( + Connections.SnapshotJournalConnectionCleanupError + ) + await expect(f.storage.awaitSnapshotJournalMaintenanceCleanup()).rejects.toBeInstanceOf( + Connections.SnapshotJournalConnectionCleanupError + ) + await expect(f.storage.maintainSnapshotJournal(f.request)).rejects.toThrow('destruction') + await expect(f.storage.destroy()).rejects.toBeInstanceOf(Connections.SnapshotJournalConnectionCleanupError) + } finally { + jest.restoreAllMocks() + await f.storage.destroy().catch(() => undefined) + await rm((f.k.client.config.connection as Knex.Sqlite3ConnectionConfig).filename.replace(/\/wallet\.sqlite$/, ''), { + recursive: true, + force: true + }) + } +}) + +function mysqlOwner() { + let completed = false + const execution = Promise.resolve() + const trx = { + executionPromise: execution, + isCompleted: jest.fn(() => completed), + commit: jest.fn(async () => { + completed = true + }), + rollback: jest.fn(async () => { + completed = true + }) + } + const writer = { transaction: jest.fn(async () => trx), destroy: jest.fn(async () => undefined) }, + reader = { destroy: jest.fn(async () => undefined) } + const factory = jest.requireActual<{ knex: typeof knex }>('knex') + jest + .spyOn(factory, 'knex') + .mockReturnValueOnce(writer as unknown as Knex) + .mockReturnValueOnce(reader as unknown as Knex) + jest.spyOn(Backend, 'prepareSnapshotJournalCaptureBackend').mockResolvedValue({ kind: 'mysql' }) + jest.spyOn(Backend, 'bindSnapshotJournalCaptureBackend').mockResolvedValue('a'.repeat(64)) + jest + .spyOn(Connections, 'withSnapshotJournalConnections') + .mockImplementation(async (_writer, _reader, active, run) => { + active() + return await run({}, {}) + }) + jest.spyOn(Fence, 'lockSnapshotJournalMaintenanceOwner').mockResolvedValue() + jest.spyOn(Mysql, 'readSnapshotJournalMysqlGeneration').mockResolvedValue({ + epoch, + source: 'a'.repeat(64), + plan: 'b'.repeat(64), + ceiling, + complete: true, + enabled: true, + nextObject: 59 + }) + jest.spyOn(Receipts, 'advanceSnapshotJournalFloor').mockResolvedValue({ + floor: snapshotJournalRevision('0'), + highWater: snapshotJournalRevision('1'), + examined: 0, + liveReceipts: 0 + }) + const config: Knex.Config = { + client: 'mysql2', + connection: { database: 'synthetic' }, + pool: { min: 8, max: 9 }, + acquireConnectionTimeout: 12000 + } + return { trx, writer, reader, config, factory } +} +test('MySQL setup preserves connection descriptors and bounds owned pool capacity and acquisition', async () => { + const m = mysqlOwner() + Object.defineProperty(m.config.connection, 'password', { enumerable: false, get: () => 'synthetic-secret' }) + const task = maintainSnapshotJournal(async () => m.config, input()) + expect(await task.result).toMatchObject({ kind: 'floor', value: { highWater: '1' } }) + await task.closed + for (const [config] of (m.factory.knex as unknown as jest.Mock).mock.calls) { + expect(config.pool).toMatchObject({ min: 0, max: 1 }) + expect(config.acquireConnectionTimeout).toBe(5000) + expect(Object.getOwnPropertyDescriptor(config.connection, 'password')).toEqual( + Object.getOwnPropertyDescriptor(m.config.connection, 'password') + ) + } + expect(m.config.pool).toMatchObject({ min: 8, max: 9 }) + expect(m.trx.commit).toHaveBeenCalledTimes(1) + expect(m.trx.rollback).not.toHaveBeenCalled() + expect(m.writer.destroy).toHaveBeenCalledTimes(1) + expect(m.reader.destroy).toHaveBeenCalledTimes(1) +}) +test.each(['backend', 'begin', 'generation', 'operation', 'commit', 'execution'] as const)( + 'retains %s failure and drains owned pools', + async phase => { + const m = mysqlOwner(), + failure = new Error('source ' + phase) + if (phase === 'backend') jest.spyOn(Backend, 'bindSnapshotJournalCaptureBackend').mockRejectedValue(failure) + else if (phase === 'begin') m.writer.transaction.mockRejectedValue(failure) + else if (phase === 'generation') jest.spyOn(Mysql, 'readSnapshotJournalMysqlGeneration').mockRejectedValue(failure) + else if (phase === 'operation') jest.spyOn(Receipts, 'advanceSnapshotJournalFloor').mockRejectedValue(failure) + else if (phase === 'commit') m.trx.commit.mockRejectedValue(failure) + else { + m.trx.executionPromise = Promise.reject(failure) + void m.trx.executionPromise.catch(() => undefined) + } + const task = maintainSnapshotJournal(async () => m.config, input()) + await expect(task.result).rejects.toBe(failure) + await expect(task.closed).rejects.toBe(failure) + expect(m.writer.destroy).toHaveBeenCalledTimes(1) + expect(m.reader.destroy).toHaveBeenCalledTimes(1) + } +) +test('rollback failure retains the operation and cleanup causes', async () => { + const m = mysqlOwner(), + source = new Error('source failure'), + cleanup = new Error('rollback failure') + jest.spyOn(Receipts, 'advanceSnapshotJournalFloor').mockRejectedValue(source) + m.trx.rollback.mockRejectedValue(cleanup) + const task = maintainSnapshotJournal(async () => m.config, input()) + const error = await task.result.then( + () => { + throw new Error('Expected cleanup failure') + }, + value => value as Connections.SnapshotJournalConnectionCleanupError + ) + expect(error).toBeInstanceOf(Connections.SnapshotJournalConnectionCleanupError) + expect((error.cause as AggregateError).errors).toEqual([source, cleanup]) + await expect(task.closed).rejects.toBe(error) +}) +test('pool destruction failure retains an undefined source rejection and its native cause', async () => { + const m = mysqlOwner(), + cleanup = new Error('pool destroy failure') + jest.spyOn(Receipts, 'advanceSnapshotJournalFloor').mockRejectedValue(undefined) + m.writer.destroy.mockRejectedValue(cleanup) + const task = maintainSnapshotJournal(async () => m.config, input()) + const error = await task.result.then( + () => { + throw new Error('Expected cleanup failure') + }, + value => value as Connections.SnapshotJournalConnectionCleanupError + ) + expect(error).toBeInstanceOf(Connections.SnapshotJournalConnectionCleanupError) + expect((error.cause as AggregateError).errors).toEqual([undefined, cleanup]) + await expect(task.closed).rejects.toBe(error) +}) +test('missing operation result fails closed after native providers drain', async () => { + const m = mysqlOwner() + jest.spyOn(Connections, 'withSnapshotJournalConnections').mockResolvedValue(undefined) + const task = maintainSnapshotJournal(async () => m.config, input()) + await expect(task.result).rejects.toThrow('generation') + await expect(task.closed).rejects.toThrow('generation') + expect(m.writer.destroy).toHaveBeenCalledTimes(1) +}) +test('cleanup-only pool failure cannot publish an otherwise committed result', async () => { + const m = mysqlOwner(), + cleanup = new Error('native pool closure failed') + m.reader.destroy.mockRejectedValue(cleanup) + const task = maintainSnapshotJournal(async () => m.config, input()) + const error = await task.result.then( + () => { + throw new Error('Expected cleanup failure') + }, + value => value as Connections.SnapshotJournalConnectionCleanupError + ) + expect((error.cause as AggregateError).errors).toEqual([cleanup]) + await expect(task.closed).rejects.toBe(error) + expect(m.trx.commit).toHaveBeenCalledTimes(1) +}) +test('a driver that still reports an active transaction must drain it before returning', async () => { + const m = mysqlOwner(), + cleanup = new Error('active transaction cleanup failed') + m.trx.isCompleted.mockReturnValue(false) + m.trx.rollback.mockRejectedValue(cleanup) + const task = maintainSnapshotJournal(async () => m.config, input()) + const error = await task.result.then( + () => { + throw new Error('Expected cleanup failure') + }, + value => value as Connections.SnapshotJournalConnectionCleanupError + ) + expect((error.cause as AggregateError).errors).toEqual([cleanup]) + await expect(task.closed).rejects.toBe(error) + expect(m.trx.commit).toHaveBeenCalledTimes(1) + expect(m.trx.rollback).toHaveBeenCalledTimes(1) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenance.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenance.ts new file mode 100644 index 000000000..e7150957d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenance.ts @@ -0,0 +1,195 @@ +import { knex as createKnex, type Knex } from 'knex' +import { WERR_INVALID_OPERATION, WERR_NOT_IMPLEMENTED } from '../../../sdk/WERR_errors' +import { + advanceSnapshotJournalFloor, + snapshotJournalReceiptPolicy, + type SnapshotJournalReceiptPolicy +} from './SnapshotJournalReceipt' +import { + collectSnapshotJournalTombstones, + snapshotJournalCollectionRequest, + type SnapshotJournalCollectionRequest +} from './SnapshotJournalCollection' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' +import { readSnapshotJournalSqliteGeneration } from './SnapshotJournalSqliteGeneration' +import { readSnapshotJournalMysqlGeneration } from './SnapshotJournalMysqlGeneration' +import { withSnapshotJournalConnections, SnapshotJournalConnectionCleanupError } from './SnapshotJournalConnections' +import { + prepareSnapshotJournalCaptureBackend, + bindSnapshotJournalCaptureBackend, + closeSnapshotJournalCapturePool +} from './SnapshotJournalCaptureBackend' +import { lockSnapshotJournalMaintenanceOwner } from './SnapshotJournalMaintenanceFence' +import { + runSnapshotJournalMaintenanceTask, + type SnapshotJournalMaintenanceOptions, + type SnapshotJournalMaintenanceTask +} from './SnapshotJournalMaintenanceTask' + +type Operation = + { kind: 'floor'; floor: SnapshotJournalRevision } | { kind: 'collect'; page: SnapshotJournalCollectionRequest } +export interface SnapshotJournalMaintenanceRequest { + epoch: string + ceiling: SnapshotJournalRevision + receiptPolicy: SnapshotJournalReceiptPolicy + operation: Operation +} +export type SnapshotJournalMaintenanceResult = + | { + kind: 'floor' + value: Awaited> + } + | { + kind: 'collect' + value: Awaited> + } + +function invalid(): never { + throw new WERR_INVALID_OPERATION('Snapshot journal maintenance generation is unavailable or changed') +} +function request(input: SnapshotJournalMaintenanceRequest): SnapshotJournalMaintenanceRequest { + const epoch = input.epoch, + ceiling = snapshotJournalRevision(input.ceiling), + receiptPolicy = snapshotJournalReceiptPolicy(input.receiptPolicy) + if ( + typeof epoch !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(epoch) || + ceiling === '0' + ) + return invalid() + const operation = input.operation + if (operation.kind === 'floor') + return { + epoch, + ceiling, + receiptPolicy, + operation: { + kind: 'floor', + floor: snapshotJournalRevision(operation.floor) + } + } + if (operation.kind !== 'collect') return invalid() + const page = snapshotJournalCollectionRequest(operation.page) + if (page.epoch !== epoch) return invalid() + return { + epoch, + ceiling, + receiptPolicy, + operation: { kind: 'collect', page } + } +} +function ownedConfig(config: Knex.Config): Knex.Config { + const connection = config.connection as object + return { + ...config, + connection: Object.create(Object.getPrototypeOf(connection), Object.getOwnPropertyDescriptors(connection)), + pool: { ...config.pool, min: 0, max: 1 }, + acquireConnectionTimeout: Math.min(config.acquireConnectionTimeout ?? 5000, 5000) + } +} +async function operation(k: Knex, input: SnapshotJournalMaintenanceRequest): Promise { + if (input.operation.kind === 'floor') + return { + kind: 'floor', + value: await advanceSnapshotJournalFloor(k, input.operation.floor) + } + return { + kind: 'collect', + value: await collectSnapshotJournalTombstones(k, input.operation.page) + } +} + +/** One global, bounded internal maintenance operation on a complete owned generation. + * No source rows or physical files are removed. Cancellation rejects the result + * promptly; closed owns all native cleanup and determines provider admission. + * Raw operator DDL is outside the configured Knex migration-owner contract. + * This neither registers a journal nor advertises an incremental capability. + */ +export function maintainSnapshotJournal( + resolveConfig: () => Promise, + input: SnapshotJournalMaintenanceRequest, + options: SnapshotJournalMaintenanceOptions = {} +): SnapshotJournalMaintenanceTask { + const detached = request(input) + return runSnapshotJournalMaintenanceTask(async assertActive => { + let writer: Knex | undefined, verifier: Knex | undefined, result: SnapshotJournalMaintenanceResult | undefined + let failure: { error: unknown } | undefined + try { + const config = await resolveConfig() + assertActive() + if (config === undefined) + throw new WERR_NOT_IMPLEMENTED('Snapshot journal maintenance requires file-backed SQLite WAL or static MySQL') + const expected = await prepareSnapshotJournalCaptureBackend(config) + assertActive() + writer = createKnex(ownedConfig(config)) + verifier = createKnex(ownedConfig(config)) + result = await withSnapshotJournalConnections( + writer, + verifier, + assertActive, + async (write, read) => { + await bindSnapshotJournalCaptureBackend(writer!, write, verifier!, read, expected) + assertActive() + if (expected.kind === 'sqlite') await writer!.raw('PRAGMA busy_timeout = 0').connection(write) + const trx = await writer!.transaction({ connection: write }) + let operationFailure: { error: unknown } | undefined + let value: SnapshotJournalMaintenanceResult | undefined + void trx.executionPromise.catch(() => undefined) + try { + assertActive() + await lockSnapshotJournalMaintenanceOwner(trx, config.migrations) + assertActive() + const generation = + expected.kind === 'sqlite' + ? await readSnapshotJournalSqliteGeneration(trx, detached.receiptPolicy, config.migrations) + : await readSnapshotJournalMysqlGeneration( + trx, + detached.ceiling, + detached.receiptPolicy, + config.migrations + ) + if (!generation.complete || generation.epoch !== detached.epoch || generation.ceiling !== detached.ceiling) + return invalid() + assertActive() + value = await operation(trx, detached) + assertActive() + await trx.commit() + await trx.executionPromise + } catch (error) { + operationFailure = { error } + } + if (!trx.isCompleted()) { + try { + await trx.rollback() + await trx.executionPromise + } catch (error) { + throw new SnapshotJournalConnectionCleanupError( + new AggregateError( + [...(operationFailure === undefined ? [] : [operationFailure.error]), error], + 'Snapshot journal maintenance transaction did not drain' + ) + ) + } + } + if (operationFailure !== undefined) throw operationFailure.error + return value + }, + closeSnapshotJournalCapturePool + ) + } catch (error) { + failure = { error } + } + const settled = await Promise.allSettled([writer?.destroy(), verifier?.destroy()]) + const failed = settled.filter(value => value.status === 'rejected') + if (failed.length) + throw new SnapshotJournalConnectionCleanupError( + new AggregateError( + [...(failure === undefined ? [] : [failure.error]), ...failed.map(value => value.reason)], + 'Snapshot journal maintenance owned pools did not close' + ) + ) + if (failure !== undefined) throw failure.error + if (result === undefined) return invalid() + return result + }, options) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.test.ts new file mode 100644 index 000000000..0686a9cca --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.test.ts @@ -0,0 +1,245 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { lockSnapshotJournalMaintenanceOwner as lock } from './SnapshotJournalMaintenanceFence' +import { SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL } from './SnapshotJournalSqliteClock' + +async function fixture() { + const directory = await mkdtemp(join(tmpdir(), 'ts569-maintenance-owner-')) + const config: Knex.Config = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + } + const k = knex(config), + peer = knex(config) + await k.raw('PRAGMA journal_mode=WAL') + await k.raw('PRAGMA busy_timeout=0') + await peer.raw('PRAGMA busy_timeout=0') + await k.raw(SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL) + await k('snapshot_journal_clock').insert({ id: 1, revision: 0, ceiling: 10000, enabled: 1, reason: null }) + await k.schema.createTable('knex_migrations_lock', table => { + table.increments('index') + table.integer('is_locked').notNullable() + }) + await k('knex_migrations_lock').insert({ is_locked: 0 }) + return { + k, + peer, + async close() { + await Promise.all([k.destroy(), peer.destroy()]) + await rm(directory, { recursive: true, force: true }) + } + } +} +afterEach(() => jest.restoreAllMocks()) +test('requires a caller transaction and an approved backend before any mutation', async () => { + const f = await fixture() + try { + await expect(lock(f.k)).rejects.toThrow('migration owner') + await expect( + lock({ isTransaction: true, client: { config: { client: 'pg' } } } as unknown as Knex) + ).rejects.toThrow('migration owner') + expect((await f.k('snapshot_journal_clock').first()).revision).toBe(0) + } finally { + await f.close() + } +}) +test('reserves WAL writing without consuming an event or changing the Knex owner', async () => { + const f = await fixture() + try { + const before = await f.k('knex_migrations_lock') + await f.k.transaction(trx => lock(trx)) + expect(await f.k('knex_migrations_lock')).toEqual(before) + expect((await f.k('snapshot_journal_clock').first()).revision).toBe(0) + } finally { + await f.close() + } +}) +test.each(['claimed', 'missing', 'duplicate', 'unsafe index'] as const)( + 'refuses %s migration ownership atomically', + async state => { + const f = await fixture() + try { + if (state === 'claimed') await f.k('knex_migrations_lock').update({ is_locked: 1 }) + else if (state === 'missing') await f.k('knex_migrations_lock').delete() + else if (state === 'duplicate') await f.k('knex_migrations_lock').insert({ is_locked: 0 }) + else await f.k('knex_migrations_lock').update({ index: 0 }) + await expect(f.k.transaction(trx => lock(trx))).rejects.toThrow('migration owner') + expect((await f.k('snapshot_journal_clock').first()).revision).toBe(0) + } finally { + await f.close() + } + } +) +test('configured migration table and schema use their actual owner', async () => { + const f = await fixture() + try { + await f.k.schema.renameTable('knex_migrations_lock', 'custom_migrations_lock') + await f.k.transaction(trx => lock(trx, { tableName: 'custom_migrations', schemaName: 'main' })) + await expect(f.k.transaction(trx => lock(trx))).rejects.toThrow() + for (const config of [ + { tableName: '' }, + { tableName: 'x'.repeat(513) }, + { schemaName: '' }, + { schemaName: 'x'.repeat(65) } + ]) { + await expect(f.k.transaction(trx => lock(trx, config))).rejects.toThrow('migration owner') + } + expect((await f.k('snapshot_journal_clock').first()).revision).toBe(0) + } finally { + await f.close() + } +}) +test('held WAL writer refuses immediately instead of queuing maintenance', async () => { + const f = await fixture(), + held = await f.peer.transaction() + try { + await lock(held) + await expect(f.k.transaction(trx => lock(trx))).rejects.toMatchObject({ code: 'SQLITE_BUSY' }) + } finally { + await held.rollback() + await f.close() + } +}) +test.each(['busy timeout', 'journal mode', 'clock missing'] as const)('refuses unsafe %s', async state => { + const f = await fixture() + try { + if (state === 'busy timeout') await f.k.raw('PRAGMA busy_timeout=1') + else if (state === 'journal mode') { + await f.peer.destroy() + await f.k.raw('PRAGMA journal_mode=DELETE') + } else await f.k('snapshot_journal_clock').delete() + await expect(f.k.transaction(trx => lock(trx))).rejects.toThrow('migration owner') + } finally { + await f.close() + } +}) +test('MySQL current owner lock is NOWAIT and reserves no global clock during metadata checks', async () => { + const calls: string[] = [] + let rows: Array<{ index: unknown; is_locked: unknown }> = [{ index: 1, is_locked: 0 }] + const query = { + select(...columns: string[]) { + calls.push('select:' + columns.join(',')) + return this + }, + limit(value: number) { + calls.push('limit:' + value) + return this + }, + withSchema(value: string) { + calls.push('schema:' + value) + return this + }, + forUpdate() { + calls.push('current') + return this + }, + noWait() { + calls.push('nowait') + return this + } + } + const k = Object.assign( + (name: string) => { + calls.push(name) + return Object.assign(Promise.resolve(rows), query) + }, + { isTransaction: true, client: { config: { client: 'mysql2' } } } + ) as unknown as Knex + await lock(k, { tableName: 'x'.repeat(59), schemaName: 's'.repeat(64) }) + expect(calls).toEqual([ + 'x'.repeat(59) + '_lock', + 'select:index,is_locked', + 'limit:2', + 'schema:' + 's'.repeat(64), + 'current', + 'nowait' + ]) + calls.length = 0 + k.client.config.client = 'mysql' + await lock(k) + expect(calls).toEqual(['knex_migrations_lock', 'select:index,is_locked', 'limit:2', 'current', 'nowait']) + for (const config of [ + { tableName: { length: 1, toString: () => 'knex_migrations' } }, + { schemaName: ['synthetic'] } + ]) { + calls.length = 0 + await expect(lock(k, config as Knex.MigratorConfig)).rejects.toThrow('migration owner') + expect(calls).toHaveLength(0) + } + await expect(lock(k, { tableName: 'x'.repeat(60) })).rejects.toThrow('migration owner') + for (const value of ['1', 0, -1, Number.MAX_SAFE_INTEGER + 1]) { + rows = [{ index: value, is_locked: 0 }] + await expect(lock(k)).rejects.toThrow('migration owner') + } + rows = [{ index: 1, is_locked: '0' }] + await expect(lock(k)).rejects.toThrow('migration owner') +}) + +test.each(['timeout missing', 'timeout duplicate', 'mode missing', 'mode duplicate'] as const)( + 'refuses malformed SQLite %s responses before reserving its writer', + async state => { + const query = jest.fn() + const k = Object.assign(query, { + isTransaction: true, + client: { config: { client: 'better-sqlite3' } }, + raw: jest.fn(async (sql: string) => { + if (sql === 'PRAGMA busy_timeout') + return state === 'timeout missing' + ? [] + : Array.from({ length: state === 'timeout duplicate' ? 2 : 1 }, () => ({ timeout: 0 })) + return state === 'mode missing' + ? [] + : Array.from({ length: state === 'mode duplicate' ? 2 : 1 }, () => ({ journal_mode: 'wal' })) + }) + }) as unknown as Knex + await expect(lock(k)).rejects.toThrow('migration owner') + expect(query).not.toHaveBeenCalled() + } +) + +test('the sqlite3 prerequisite selects the exact owner fields after writer reservation', async () => { + const calls: string[] = [] + const query = { + where(id: string, value: number) { + calls.push('where:' + id + ':' + value) + return this + }, + async update() { + calls.push('reserve') + return 1 + }, + select(...columns: string[]) { + calls.push('select:' + columns.join(',')) + return this + }, + limit(value: number) { + calls.push('limit:' + value) + return this + } + } + const k = Object.assign( + (table: string) => { + calls.push(table) + return Object.assign(Promise.resolve([{ index: 1, is_locked: 0 }]), query) + }, + { + isTransaction: true, + client: { config: { client: 'sqlite3' } }, + ref: (name: string) => name, + raw: async (sql: string) => (sql === 'PRAGMA busy_timeout' ? [{ timeout: 0 }] : [{ journal_mode: 'wal' }]) + } + ) as unknown as Knex + await lock(k, { tableName: 'x'.repeat(512) }) + expect(calls).toEqual([ + 'snapshot_journal_clock', + 'where:id:1', + 'reserve', + 'x'.repeat(512) + '_lock', + 'select:index,is_locked', + 'limit:2' + ]) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.ts new file mode 100644 index 000000000..970cd6608 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.ts @@ -0,0 +1,55 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' + +function invalid(): never { + throw new WERR_INVALID_OPERATION('Snapshot journal maintenance cannot exclude the configured migration owner') +} + +/** Internal prerequisite for complete owned-generation validation and one bounded + * maintenance operation. Use a fresh caller-owned transaction. MySQL locks only + * the migration owner during metadata validation; foreground clock writers can + * proceed until the bounded floor/page operation reserves its current clock. + * SQLite reserves the writer before any read can pin an old snapshot. Neither + * reservation allocates an event position, and the owner row remains unchanged. + * Every approved schema operation must use that migration owner. This does not + * exclude raw operator DDL, publish a migration, or authorize source-row removal. + */ +export async function lockSnapshotJournalMaintenanceOwner(k: Knex, config?: Knex.MigratorConfig): Promise { + const driver = k.client.config.client + const sqlite = driver === 'better-sqlite3' || driver === 'sqlite3' + if (!k.isTransaction || (!sqlite && driver !== 'mysql' && driver !== 'mysql2')) return invalid() + const table = config?.tableName ?? 'knex_migrations', + schema = config?.schemaName + if ( + typeof table !== 'string' || + table.length === 0 || + table.length > (sqlite ? 512 : 59) || + (schema !== undefined && (typeof schema !== 'string' || schema.length === 0 || schema.length > 64)) + ) + return invalid() + if (sqlite) { + const timeouts: Array<{ timeout: number }> = await k.raw('PRAGMA busy_timeout') + const modes: Array<{ journal_mode: string }> = await k.raw('PRAGMA journal_mode') + if (timeouts.length !== 1 || timeouts[0].timeout !== 0 || modes.length !== 1 || modes[0].journal_mode !== 'wal') + return invalid() + if ( + (await k('snapshot_journal_clock') + .where('id', 1) + .update({ revision: k.ref('revision') })) !== 1 + ) + return invalid() + } + const query = k(table + '_lock') + .select('index', 'is_locked') + .limit(2) + if (schema !== undefined) query.withSchema(schema) + if (!sqlite) query.forUpdate().noWait() + const rows: Array<{ index: unknown; is_locked: unknown }> = await query + if ( + rows.length !== 1 || + !Number.isSafeInteger(rows[0].index) || + (rows[0].index as number) < 1 || + rows[0].is_locked !== 0 + ) + return invalid() +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.test.ts new file mode 100644 index 000000000..4e5e3991b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.test.ts @@ -0,0 +1,334 @@ +import { getEventListeners } from 'node:events' +import { runSnapshotJournalMaintenanceTask as run } from './SnapshotJournalMaintenanceTask' +import { WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { SnapshotCancelledError } from '../SnapshotCancelledError' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' + +function gate() { + let resolve!: () => void + let reject!: (error: unknown) => void + const promise = new Promise((yes, no) => { + resolve = yes + reject = no + }) + return { promise, resolve, reject } +} +async function pending(promise: Promise) { + let settled = false + void promise.then( + () => { + settled = true + }, + () => { + settled = true + } + ) + await new Promise(resolve => setImmediate(resolve)) + expect(settled).toBe(false) +} +afterEach(() => { + jest.useRealTimers() + jest.restoreAllMocks() +}) + +test('invalid lifetimes refuse before allocating native work', () => { + for (const lifetimeMs of [0, -1, 1.5, NaN, Infinity, 30001, '100']) { + let allocated = false + expect(() => + run( + async () => { + allocated = true + }, + { lifetimeMs: lifetimeMs as number } + ) + ).toThrow(WERR_INVALID_PARAMETER) + expect(() => run(async () => undefined, { lifetimeMs: lifetimeMs as number })).toThrow( + 'The lifetimeMs parameter must be an integer from 1 to 30000' + ) + expect(allocated).toBe(false) + } +}) +test('already aborted work never enters the native callback', async () => { + jest.useFakeTimers() + let allocated = false + const task = run( + async () => { + allocated = true + }, + { signal: AbortSignal.abort() } + ) + await expect(task.result).rejects.toBeInstanceOf(SnapshotCancelledError) + await task.closed + await task.close() + expect(allocated).toBe(false) + expect(jest.getTimerCount()).toBe(0) +}) +test.each([1, 30000])('admits exact lifetime boundary %i and releases its timer and listener', async lifetimeMs => { + jest.useFakeTimers() + const controller = new AbortController() + const task = run(async () => 23, { signal: controller.signal, lifetimeMs }) + expect(jest.getTimerCount()).toBe(1) + expect(getEventListeners(controller.signal, 'abort')).toHaveLength(1) + expect(await task.result).toBe(23) + await task.closed + expect(jest.getTimerCount()).toBe(0) + expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0) + await Promise.all([task.close(), task.close()]) + controller.abort() + expect(await task.result).toBe(23) + expect(jest.getTimerCount()).toBe(0) +}) +test('cancellation detaches result-lifetime resources while native cleanup remains owned', async () => { + jest.useFakeTimers() + const controller = new AbortController(), + entered = gate(), + drain = gate() + let physicallyClosed = false + const task = run( + async active => { + entered.resolve() + await drain.promise + physicallyClosed = true + active() + }, + { signal: controller.signal } + ) + await entered.promise + expect(jest.getTimerCount()).toBe(1) + controller.abort() + await expect(task.result).rejects.toBeInstanceOf(SnapshotCancelledError) + expect(jest.getTimerCount()).toBe(0) + expect(getEventListeners(controller.signal, 'abort')).toHaveLength(0) + expect(physicallyClosed).toBe(false) + drain.resolve() + await task.closed + expect(physicallyClosed).toBe(true) +}) +test('the first cancellation reason remains exact through repeated close and native drain', async () => { + const controller = new AbortController(), + entered = gate(), + drain = gate() + let nativeStop: unknown + const task = run( + async active => { + entered.resolve() + await drain.promise + try { + active() + } catch (error) { + nativeStop = error + throw error + } + }, + { signal: controller.signal } + ) + await entered.promise + controller.abort() + const reason = await task.result.catch(error => error as unknown) + expect(reason).toBeInstanceOf(SnapshotCancelledError) + const closing = Promise.all([task.close(), task.close()]) + drain.resolve() + await closing + expect(nativeStop).toBe(reason) + expect(await task.result.catch(error => error as unknown)).toBe(reason) +}) +test('normal results and closure wait for physical cleanup and close remains idempotent', async () => { + const cleanup = gate() + const task = run(async () => { + await cleanup.promise + return 23 + }) + await pending(task.result) + await pending(task.closed) + cleanup.resolve() + expect(await task.result).toBe(23) + await task.closed + await Promise.all([task.close(), task.close()]) +}) +test('cancellation rejects promptly but native drain owns closure', async () => { + const cleanup = gate(), + entered = gate(), + controller = new AbortController() + const task = run( + async active => { + entered.resolve() + await cleanup.promise + active() + return 'late' + }, + { signal: controller.signal } + ) + await entered.promise + controller.abort() + await expect(task.result).rejects.toBeInstanceOf(SnapshotCancelledError) + await pending(task.closed) + const closing = task.close() + await pending(closing) + cleanup.resolve() + await closing +}) +test('expiry discards late results without releasing before native drain', async () => { + const cleanup = gate(), + entered = gate() + const task = run( + async () => { + entered.resolve() + await cleanup.promise + return 'late' + }, + { lifetimeMs: 20 } + ) + await entered.promise + const error = await task.result.catch(error => error as unknown) + expect(error).toBeInstanceOf(SnapshotResourceLimitError) + expect(error).toMatchObject({ message: 'Snapshot journal maintenance expired' }) + await pending(task.closed) + cleanup.resolve() + await task.closed +}) +test('wall and monotonic deadlines independently refuse late results', async () => { + for (const clock of ['wall', 'monotonic']) { + const wall = Date.now() + let now = wall, + monotonic = 100 + jest.spyOn(Date, 'now').mockImplementation(() => now) + jest.spyOn(performance, 'now').mockImplementation(() => monotonic) + const cleanup = gate(), + entered = gate() + const task = run( + async () => { + entered.resolve() + await cleanup.promise + return 'late' + }, + { lifetimeMs: 1000 } + ) + await entered.promise + if (clock === 'wall') now += 1000 + else { + now -= 100000 + monotonic += 1000 + } + cleanup.resolve() + const error = await task.result.catch(error => error as unknown) + expect(error).toBeInstanceOf(SnapshotResourceLimitError) + expect(error).toMatchObject({ message: 'Snapshot journal maintenance expired' }) + await task.closed + jest.restoreAllMocks() + } +}) +test('post-commit cancellation does not claim rollback of durable state', async () => { + const cleanup = gate(), + committed = gate(), + controller = new AbortController() + let floor = 0 + const task = run( + async () => { + floor = 8 + committed.resolve() + await cleanup.promise + return floor + }, + { signal: controller.signal } + ) + await committed.promise + controller.abort() + await expect(task.result).rejects.toBeInstanceOf(SnapshotCancelledError) + await pending(task.closed) + expect(floor).toBe(8) + cleanup.resolve() + await task.closed + expect(floor).toBe(8) +}) +test('unrelated native cleanup errors remain observable after cancellation', async () => { + const cleanup = gate(), + entered = gate(), + controller = new AbortController(), + error = new Error('native close failure') + const task = run( + async () => { + entered.resolve() + await cleanup.promise + }, + { signal: controller.signal } + ) + await entered.promise + controller.abort() + await expect(task.result).rejects.toBeInstanceOf(SnapshotCancelledError) + cleanup.reject(error) + await expect(task.closed).rejects.toBe(error) + await expect(task.close()).rejects.toBe(error) +}) +test('ordinary source failure including undefined cannot match an unset cancellation', async () => { + for (const error of [new Error('source failure'), undefined]) { + const task = run(async () => { + throw error + }) + const outcomes = await Promise.allSettled([task.result, task.closed]) + expect(outcomes).toEqual([ + { status: 'rejected', reason: error }, + { status: 'rejected', reason: error } + ]) + } +}) +test('early manual close refuses allocation and repeated close drains the same owner', async () => { + let allocated = false + const task = run(async () => { + allocated = true + }) + await Promise.all([task.close(), task.close()]) + await expect(task.result).rejects.toThrow('closed') + expect(allocated).toBe(false) +}) +async function executeSchedule(stage: number, index: number): Promise { + const controller = new AbortController(), + entered = gate(), + commit = gate(), + committed = gate(), + drain = gate() + let durable = false, + allocated = false + const task = run( + async active => { + allocated = true + entered.resolve() + await commit.promise + active() + durable = true + committed.resolve() + await drain.promise + return index + }, + { signal: controller.signal } + ) + if (stage === 0) controller.abort() + else { + await entered.promise + if (stage === 1) controller.abort() + commit.resolve() + if (stage >= 2) { + await committed.promise + if (stage === 2) controller.abort() + } + if (stage !== 1) await pending(task.closed) + } + drain.resolve() + const outcome = await task.result.then( + value => ({ value }), + error => ({ error: error as unknown }) + ) + await task.closed + if (stage === 3) expect(outcome).toEqual({ value: index }) + else expect(outcome).toEqual({ error: expect.any(SnapshotCancelledError) }) + expect(allocated).toBe(stage !== 0) + expect(durable).toBe(stage >= 2) +} + +test('300 seeded cancellation/commit/drain schedules retain ownership', async () => { + let seed = 3242026 + const random = () => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0 + return seed + } + for (let index = 0; index < 300; index++) await executeSchedule(random() % 4, index) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.ts new file mode 100644 index 000000000..af360a267 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.ts @@ -0,0 +1,91 @@ +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../../sdk/WERR_errors' +import { SnapshotCancelledError } from '../SnapshotCancelledError' +import { SnapshotResourceLimitError } from '../SnapshotResourceLimitError' + +export interface SnapshotJournalMaintenanceOptions { + /** Total admission/result lifetime; physical native cleanup must still drain. */ + lifetimeMs?: number + signal?: AbortSignal +} +export interface SnapshotJournalMaintenanceTask { + result: Promise + closed: Promise + close: () => Promise +} +function deferred() { + let resolve!: (value: T) => void, reject!: (reason: unknown) => void + const promise = new Promise((yes, no) => { + resolve = yes + reject = no + }) + return { promise, resolve, reject } +} + +/** Private owner lifecycle. The run callback must await every native operation + * and all physical cleanup. Rejecting result never releases the provider slot; + * only closed permits admission. A cancellation after commit discards the late + * result without claiming rollback; floor/page retry uses database state. + */ +export function runSnapshotJournalMaintenanceTask( + run: (assertActive: () => void) => Promise, + options: SnapshotJournalMaintenanceOptions = {} +): SnapshotJournalMaintenanceTask { + const lifetimeMs = options.lifetimeMs ?? 5000, + signal = options.signal + if (!Number.isSafeInteger(lifetimeMs) || lifetimeMs < 1 || lifetimeMs > 30000) + throw new WERR_INVALID_PARAMETER('lifetimeMs', 'an integer from 1 to 30000') + const result = deferred(), + closed = deferred() + const expiresAt = Date.now() + lifetimeMs, + started = performance.now() + let stopReason: WERR_INVALID_OPERATION | undefined, + finished = false + let timer: ReturnType | undefined + void result.promise.catch(() => undefined) + void closed.promise.catch(() => undefined) + const detach = () => { + if (timer !== undefined) clearTimeout(timer) + signal?.removeEventListener('abort', abort) + } + const stop = (reason: WERR_INVALID_OPERATION) => { + if (finished || stopReason !== undefined) return + stopReason = reason + detach() + result.reject(reason) + } + const abort = () => stop(new SnapshotCancelledError('Snapshot journal maintenance was cancelled')) + const assertActive = () => { + if (Date.now() >= expiresAt || performance.now() - started >= lifetimeMs) + stop(new SnapshotResourceLimitError('Snapshot journal maintenance expired')) + if (stopReason !== undefined) throw stopReason + } + const finish = async () => { + try { + assertActive() + const value = await run(assertActive) + assertActive() + closed.resolve() + result.resolve(value) + } catch (error) { + if (stopReason !== undefined && error === stopReason) closed.resolve() + else closed.reject(error) + result.reject(error) + } finally { + finished = true + detach() + } + } + signal?.addEventListener('abort', abort, { once: true }) + if (signal?.aborted) abort() + if (stopReason === undefined) + timer = setTimeout(() => stop(new SnapshotResourceLimitError('Snapshot journal maintenance expired')), lifetimeMs) + void Promise.resolve().then(finish) + return { + result: result.promise, + closed: closed.promise, + async close() { + stop(new WERR_INVALID_OPERATION('Snapshot journal maintenance is closed')) + await closed.promise + } + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlBootstrap.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlBootstrap.test.ts new file mode 100644 index 000000000..10ab1e09c --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlBootstrap.test.ts @@ -0,0 +1,694 @@ +import { knex, type Knex } from 'knex' +import { copySnapshotJournalBootstrapPage, SNAPSHOT_JOURNAL_BOOTSTRAP_DDL } from './SnapshotJournalBootstrap' +import { installSnapshotJournalMysqlClock } from './SnapshotJournalMysqlClock' +import { SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL } from './SnapshotJournalSqliteObservers' +import { readSnapshotJournalMetadataPage } from './SnapshotJournalPage' +import { snapshotJournalRevision as rev } from './SnapshotJournalRevision' +import { numeric, legacyNames } from '../../schema/snapshotSqliteMembership' + +// Real Knex MySQL compilation/response processing with relational state and +// rollback in SQLite. Native MySQL fixtures independently prove DDL, locks, +// source observers, optimizer plans and process/server-crash behavior. +const identities = [ + ...numeric.map(source => ({ + table: source.table, + keys: [source.key], + text: undefined as string | undefined, + extra: undefined as string | undefined + })), + { + table: 'tx_labels_map', + keys: ['txLabelId', 'transactionId'], + text: undefined, + extra: undefined + }, + { + table: 'output_tags_map', + keys: ['outputTagId', 'outputId'], + text: undefined, + extra: undefined + }, + { + table: 'certificate_fields', + keys: ['fieldName', 'certificateId'], + text: 'fieldName', + extra: undefined + }, + { + table: legacyNames.profile, + keys: ['snapshotTableId', 'snapshotUserId', 'snapshotRowId'], + text: undefined, + extra: undefined + }, + { + table: legacyNames.relation, + keys: ['snapshotTableId', 'snapshotUserId', 'snapshotLeftId', 'snapshotRightId'], + text: undefined, + extra: 'snapshotMembership' + }, + { + table: legacyNames.certificate, + keys: ['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId'], + text: 'snapshotFieldName', + extra: 'snapshotMembership' + }, + { + table: legacyNames.keys, + keys: ['tableId', 'userId', 'rowId'], + text: undefined, + extra: 'present' + } +] +interface Query { + sql: string + bindings: Knex.RawBinding[] + method: string + response?: unknown +} +async function driver() { + const db = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }), + k = knex({ client: 'mysql2' }) + const state = { + next: 9007199254740993n, + last: '', + allocated: undefined as unknown, + filesort: false, + noIndex: false, + oversized: false, + failMetadata: false, + missingRevisionClock: false, + nonBinaryText: false, + invalidIdentity: undefined as unknown + } + const queries: Query[] = [] + const connection = { + __knexUid: 'journal-driver', + query: ( + options: { sql: string }, + bindings: Knex.RawBinding[] | undefined, + callback: (error: unknown, rows?: unknown, fields?: unknown) => void + ) => { + void k.client._query(connection, { sql: options.sql, bindings: bindings ?? [], method: 'raw' }).then( + (result: { response: [unknown, unknown] }) => callback(null, ...result.response), + (error: unknown) => callback(error) + ) + } + } + k.client.acquireConnection = async () => connection + k.client.releaseConnection = async () => undefined + k.client._query = async (_connection: unknown, q: Query) => { + queries.push({ sql: q.sql, bindings: q.bindings, method: q.method }) + const respond = (value: unknown) => { + q.response = [value, []] + return q + } + if (q.sql.startsWith('SELECT INDEX_NAME name,COLUMN_NAME field')) { + const source = identities.find(source => source.table === q.bindings[0]) + if (!source) throw new Error('Unknown fixture index') + return respond(state.noIndex ? [] : source.keys.map(field => ({ name: 'PRIMARY', field }))) + } + if (q.sql === 'SELECT CAST(LAST_INSERT_ID() AS CHAR) value') + return respond([{ value: state.allocated === undefined ? state.last : state.allocated }]) + if (state.missingRevisionClock && q.sql.startsWith('select CAST(') && q.sql.includes('`snapshot_journal_clock`')) + return respond([]) + let sql = q.sql + .replace(/ FORCE INDEX \(`[^`]+`\)/g, '') + .replace(/ (?:for (?:share|update)(?: nowait)?|lock in share mode)$/i, '') + .replaceAll(' AS BINARY)', ' AS BLOB)') + .replaceAll(' AS SIGNED)', ' AS INTEGER)') + if (sql.startsWith('EXPLAIN ')) { + const plan: Array<{ detail: string }> = await db.raw('EXPLAIN QUERY PLAN ' + sql.slice(8), q.bindings) + return respond( + plan.map(row => ({ + Extra: state.filesort ? 'Using filesort' : row.detail.includes('TEMP B-TREE') ? 'Using filesort' : '' + })) + ) + } + if (sql.startsWith('CREATE TABLE ')) { + sql = sql + .replace(/ ENGINE=InnoDB$/, '') + .replace('revision BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY', 'revision TEXT NOT NULL PRIMARY KEY') + .replaceAll('BIGINT UNSIGNED', 'INTEGER') + } + if (/^insert into `snapshot_journal_events` \(\) values \(\)$/i.test(sql)) { + state.last = String(state.next++) + await db('snapshot_journal_events').insert({ revision: state.last }) + return respond({ insertId: state.last, affectedRows: 1 }) + } + sql = sql.replace(/^insert ignore into /i, 'insert or ignore into ') + if (sql.includes(' on duplicate key update ')) { + expect(sql).toMatch(/ on duplicate key update `id1` = `snapshot_journal_(?:physical|scope)`\.`id1`$/) + sql = sql.replace(/ on duplicate key update .+$/, ' on conflict do nothing') + if (state.failMetadata) throw new Error('metadata write failed') + } + const result = await db.raw(sql, q.bindings) + if (state.nonBinaryText && Array.isArray(result) && q.sql.includes('`boundedText`')) + for (const row of result) row.boundedText = 'not bytes' + if (Array.isArray(result) && sql.startsWith('select `s`.') && state.invalidIdentity !== undefined) + for (const row of result) row.transactionId = state.invalidIdentity + if (Array.isArray(result)) + return respond( + state.oversized && sql.startsWith('select `s`.') ? Array.from({ length: 257 }, () => result[0]) : result + ) + return respond({ affectedRows: result?.changes ?? 0, insertId: result?.lastInsertRowid ?? 0 }) + } + await installSnapshotJournalMysqlClock(k, rev('9223372036854775807')) + await db.raw(SNAPSHOT_JOURNAL_BOOTSTRAP_DDL) + await db('snapshot_journal_bootstrap').insert({ id: 1, stream: 0, cursor: null, rowLimit: 1000000, rowsUsed: 0 }) + for (const ddl of SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL) await db.raw(ddl) + for (const source of identities) { + const columns = [...source.keys, ...(source.extra ? [source.extra] : [])].map( + name => '`' + name + '` ' + (source.text === name ? 'TEXT' : 'INTEGER') + ' NOT NULL' + ) + await db.raw( + 'CREATE TABLE `' + + source.table + + '`(' + + columns.join(',') + + ',PRIMARY KEY(' + + source.keys.map(key => '`' + key + '`').join(',') + + '))' + ) + } + queries.length = 0 + return { + db, + k, + state, + queries, + close: async () => { + await k.destroy() + await db.destroy() + } + } +} +function sample(stream: number, id: number) { + const source = identities[stream], + row: Record = {} + for (const key of source.keys) + row[key] = + key === source.text + ? 'field-' + String(id).padStart(3, '0') + : key.toLowerCase().includes('tableid') + ? 0 + : key.toLowerCase().includes('userid') + ? 1 + : id + if (source.extra) row[source.extra] = 1 + return row +} +test.each(identities.map((source, stream) => [source.table, stream] as const))( + 'MySQL bootstrap copies and resumes indexed %s keys with exact revisions', + async (_table, stream) => { + const f = await driver() + try { + const source = identities[stream] + await f.db('snapshot_journal_bootstrap').update({ stream, cursor: null }) + await f.db(source.table).insert([sample(stream, 1), sample(stream, 2)]) + const first = await copySnapshotJournalBootstrapPage(f.k, 1000000) + expect(first).toEqual({ complete: false, selected: 2, stream, invalidated: false }) + const metadata = stream < 13 ? 'snapshot_journal_physical' : 'snapshot_journal_scope', + rows = await f.db(metadata).select('*', f.db.raw('CAST(revision AS TEXT) AS exactRevision')) + expect(rows).toHaveLength(2) + expect(rows.map(row => row.exactRevision)).toEqual(['9007199254740993', '9007199254740993']) + expect(rows.every(row => row.exactText instanceof Uint8Array)).toBe(true) + expect(await f.db('snapshot_journal_events')).toEqual([]) + await f.db(source.table).insert(sample(stream, 3)) + expect((await copySnapshotJournalBootstrapPage(f.k, 1000000)).selected).toBe(1) + const final = await copySnapshotJournalBootstrapPage(f.k, 1000000) + expect(final).toEqual({ complete: stream === 16, selected: 0, stream, invalidated: false }) + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: stream + 1, + cursor: null, + rowLimit: 1000000, + rowsUsed: 3 + }) + expect(f.queries.some(q => q.sql.endsWith('for update nowait'))).toBe(true) + expect( + f.queries.some(q => q.sql.includes('FORCE INDEX (`PRIMARY`)') && q.sql.endsWith('lock in share mode')) + ).toBe(true) + } finally { + await f.close() + } + } +) +test('MySQL bootstrap bounds each page and preserves an already observed newer generation', async () => { + const f = await driver() + try { + for (let first = 1; first <= 300; first += 50) + await f.db('transactions').insert(Array.from({ length: 50 }, (_, i) => ({ transactionId: first + i }))) + await f.db('snapshot_journal_physical').insert({ + tableId: 0, + id1: 1, + id2: 0, + exactText: Buffer.alloc(0), + revision: '9007199254740999', + generation: '9007199254740998', + present: 0 + }) + const page = await copySnapshotJournalBootstrapPage(f.k, 1000000) + expect(page.selected).toBe(256) + const writes = f.queries.filter(q => q.sql.startsWith('insert into `snapshot_journal_physical`')) + expect(writes).toHaveLength(4) + writes.forEach(query => expect(query.bindings.length).toBeLessThanOrEqual(64 * 7)) + const preserved = await f + .db('snapshot_journal_physical') + .where({ tableId: 0, id1: 1 }) + .select('present', f.db.raw('CAST(revision AS TEXT) revision'), f.db.raw('CAST(generation AS TEXT) generation')) + .first() + expect(preserved).toEqual({ + present: 0, + revision: '9007199254740999', + generation: '9007199254740998' + }) + expect((await copySnapshotJournalBootstrapPage(f.k, 1000000)).selected).toBe(44) + } finally { + await f.close() + } +}) +test.each([ + ['capacity-exhausted', '10', '11'], + ['revision-exhausted', '9223372036854775807', '9223372036854775808'] +] as const)('MySQL %s invalidates atomically without advancing bootstrap', async (reason, ceiling, next) => { + const f = await driver() + try { + await f.db('transactions').insert({ transactionId: 1 }) + await f.db('snapshot_journal_clock').update({ ceiling }) + f.state.next = BigInt(next) + expect(await copySnapshotJournalBootstrapPage(f.k, 1000000)).toEqual({ + complete: false, + selected: 1, + stream: 0, + invalidated: true + }) + expect(await f.db('snapshot_journal_invalid')).toEqual([{ id: 1, reason }]) + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect((await f.db('snapshot_journal_bootstrap').first()).cursor).toBeNull() + expect(await f.db('transactions')).toEqual([{ transactionId: 1 }]) + expect((await copySnapshotJournalBootstrapPage(f.k, 1000000)).invalidated).toBe(true) + } finally { + await f.close() + } +}) +test.each([null, 1, '-1', '1.5', 'bad'])('MySQL malformed allocator value %p rolls back page state', async value => { + const f = await driver() + try { + await f.db('transactions').insert({ transactionId: 1 }) + f.state.allocated = value + await expect(copySnapshotJournalBootstrapPage(f.k, 1000000)).rejects.toThrow('Invalid snapshot') + expect(await f.db('snapshot_journal_events')).toEqual([]) + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect((await f.db('snapshot_journal_bootstrap').first()).cursor).toBeNull() + } finally { + await f.close() + } +}) +test.each(['filesort', 'noIndex', 'oversized', 'failMetadata'] as const)( + 'MySQL %s refuses and rolls back before checkpoint advancement', + async kind => { + const f = await driver() + try { + await f.db('transactions').insert({ transactionId: 1 }) + f.state[kind] = true + await expect(copySnapshotJournalBootstrapPage(f.k, 1000000)).rejects.toThrow() + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect((await f.db('snapshot_journal_bootstrap').first()).cursor).toBeNull() + } finally { + await f.close() + } + } +) +test('MySQL oversized historical UTF-8 key invalidates without copying or changing source', async () => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update({ stream: 12 }) + await f.db('certificate_fields').insert({ fieldName: 'a'.repeat(401), certificateId: 1 }) + expect((await copySnapshotJournalBootstrapPage(f.k, 1000000)).invalidated).toBe(true) + expect(await f.db('snapshot_journal_invalid')).toEqual([{ id: 1, reason: 'key-out-of-range' }]) + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect((await f.db('certificate_fields').first()).fieldName).toHaveLength(401) + } finally { + await f.close() + } +}) +test('MySQL page continuation follows the complete byte-exact composite cursor', async () => { + const f = await driver() + try { + const values = ['A', 'a', 'a ', 'é', '中'] + for (const [i, exactText] of values.entries()) + await f.db('snapshot_journal_scope').insert({ + tableId: 12, + userId: 1, + id1: i < 3 ? 1 : 2, + id2: 0, + exactText: Buffer.from(exactText), + revision: '9007199254740993', + present: 1 + }) + let after, + actual: string[] = [] + for (let i = 0; i < 4; i++) { + const page = await readSnapshotJournalMetadataPage(f.k, { + stream: 'scope', + tableId: 12, + userId: 1, + floor: rev('0'), + low: rev('9007199254740992'), + high: rev('9007199254740999'), + limit: 2, + after + }) + actual.push(...page.rows.map(row => row.exactText)) + if (page.complete) break + after = page.after + } + expect(actual).toEqual(values) + expect(f.queries.some(q => q.sql.includes(' or (') && q.sql.includes('`j`.`exactText` > ?'))).toBe(true) + } finally { + await f.close() + } +}) +test.each(['clock', 'bootstrap'])('missing MySQL %s state refuses rather than creating a new baseline', async kind => { + const f = await driver() + try { + await f.db('snapshot_journal_' + kind).delete() + await expect(copySnapshotJournalBootstrapPage(f.k, 1000000)).rejects.toThrow('Invalid snapshot') + expect(await f.db('snapshot_journal_physical')).toEqual([]) + } finally { + await f.close() + } +}) +test.each(['not-json', '[]', '[-1]', '[1.5]', '["1"]', 'x'.repeat(2049)])( + 'invalid MySQL bootstrap cursor %s refuses without source changes', + async cursor => { + const f = await driver() + try { + await f.db('transactions').insert({ transactionId: 1 }) + await f.db('snapshot_journal_bootstrap').update({ cursor }) + await expect(copySnapshotJournalBootstrapPage(f.k, 1000000)).rejects.toThrow('Invalid snapshot') + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect(await f.db('transactions')).toEqual([{ transactionId: 1 }]) + } finally { + await f.close() + } + } +) +test('completed MySQL bootstrap is stable and a dangling terminal cursor refuses', async () => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update({ stream: 17 }) + expect(await copySnapshotJournalBootstrapPage(f.k, 1000000)).toEqual({ + complete: true, + selected: 0, + stream: 17, + invalidated: false + }) + await f.db('snapshot_journal_bootstrap').update({ cursor: '[]' }) + await expect(copySnapshotJournalBootstrapPage(f.k, 1000000)).rejects.toThrow('Invalid snapshot') + } finally { + await f.close() + } +}) + +test.each(['missingRevisionClock', 'nonBinaryText'] as const)( + 'malformed bootstrap driver %s rolls back progress', + async kind => { + const f = await driver() + try { + const stream = kind === 'nonBinaryText' ? 12 : 0 + await f.db(identities[stream].table).insert(sample(stream, 1)) + await f.db('snapshot_journal_bootstrap').update({ stream }) + const clock = await f.db('snapshot_journal_clock'), + progress = await f.db('snapshot_journal_bootstrap') + f.state[kind] = true + await expect(copySnapshotJournalBootstrapPage(f.k, 1000000)).rejects.toThrow('Invalid snapshot') + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect(await f.db('snapshot_journal_clock')).toEqual(clock) + expect(await f.db('snapshot_journal_bootstrap')).toEqual(progress) + } finally { + await f.close() + } + } +) + +test.each([14, 15, 16])('MySQL stream %s preserves absent membership and its physical-table mapping', async stream => { + const f = await driver() + try { + const source = identities[stream], + extra = source.extra! + const present = sample(stream, 1), + absent = { ...sample(stream, 2), [extra]: 0 } + if (stream === 16) absent.tableId = 1 + await f.db('snapshot_journal_bootstrap').update({ stream }) + await f.db(source.table).insert([present, absent]) + expect(await copySnapshotJournalBootstrapPage(f.k, 1000000)).toMatchObject({ + selected: 2, + invalidated: false + }) + const rows = await f.db('snapshot_journal_scope').orderBy('id1') + expect(rows.map(row => ({ tableId: row.tableId, id1: row.id1, present: row.present }))).toEqual([ + { tableId: stream === 14 ? 10 : stream === 15 ? 12 : 9, id1: 1, present: 1 }, + { tableId: stream === 14 ? 10 : stream === 15 ? 12 : 8, id1: 2, present: 0 } + ]) + } finally { + await f.close() + } +}) + +test.each(['10', '9223372036854775807'])('MySQL can allocate the exact final allowed revision %s', async ceiling => { + const f = await driver() + f.k.client.config.client = 'mysql' + try { + await f.db('transactions').insert({ transactionId: 1 }) + await f.db('snapshot_journal_clock').update({ ceiling }) + f.state.next = BigInt(ceiling) + expect(await copySnapshotJournalBootstrapPage(f.k, 1000000)).toMatchObject({ + selected: 1, + invalidated: false + }) + expect(await f.db('snapshot_journal_physical').select(f.db.raw('CAST(revision AS TEXT) revision'))).toEqual([ + { revision: ceiling } + ]) + expect(await f.db('snapshot_journal_invalid')).toEqual([]) + } finally { + await f.close() + } +}) + +test.each([-1, 0, 1.5, 'invalid', 9007199254740992])( + 'MySQL invalid historical identity %p invalidates atomically and preserves source', + async transactionId => { + const f = await driver() + try { + const malformedResponse = typeof transactionId !== 'number' || !Number.isInteger(transactionId) + await f.db('transactions').insert({ transactionId: malformedResponse ? 1 : transactionId }) + if (malformedResponse) f.state.invalidIdentity = transactionId + expect(await copySnapshotJournalBootstrapPage(f.k, 1000000)).toMatchObject({ + selected: 1, + invalidated: true + }) + expect(await f.db('snapshot_journal_invalid')).toEqual([{ id: 1, reason: 'key-out-of-range' }]) + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect((await f.db('snapshot_journal_bootstrap').first()).cursor).toBeNull() + expect((await f.db('transactions').first()).transactionId).toBe(malformedResponse ? 1 : transactionId) + } finally { + await f.close() + } + } +) + +test('MySQL accepts a 400-byte key through the final composite bootstrap cursor', async () => { + const f = await driver() + try { + const fieldName = '😀'.repeat(100) + await f.db('snapshot_journal_bootstrap').update({ stream: 12 }) + await f.db('certificate_fields').insert({ fieldName, certificateId: 1 }) + expect(await copySnapshotJournalBootstrapPage(f.k, 1000000)).toMatchObject({ + selected: 1, + invalidated: false + }) + expect(Buffer.from((await f.db('snapshot_journal_physical').first()).exactText).toString('utf8')).toBe(fieldName) + expect(await copySnapshotJournalBootstrapPage(f.k, 1000000)).toMatchObject({ + selected: 0, + invalidated: false + }) + } finally { + await f.close() + } +}) + +test('MySQL bootstrap charges an explicit immutable allowance across resumed pages', async () => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update({ rowLimit: null }) + for (let first = 1; first <= 300; first += 50) + await f.db('transactions').insert(Array.from({ length: 50 }, (_, offset) => sample(0, first + offset))) + expect(await copySnapshotJournalBootstrapPage(f.k, 300)).toMatchObject({ selected: 256, invalidated: false }) + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: '[256]', + rowLimit: 300, + rowsUsed: 256 + }) + const prior = await f.db('snapshot_journal_bootstrap').first() + await expect(copySnapshotJournalBootstrapPage(f.k, 301)).rejects.toThrow('Invalid snapshot') + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual(prior) + expect(await copySnapshotJournalBootstrapPage(f.k, 300)).toMatchObject({ selected: 44, invalidated: false }) + const full = await f.db('snapshot_journal_bootstrap').first() + expect(full).toEqual({ id: 1, stream: 0, cursor: '[300]', rowLimit: 300, rowsUsed: 300 }) + await f.db('transactions').insert(sample(0, 301)) + expect(await copySnapshotJournalBootstrapPage(f.k, 300)).toEqual({ + complete: false, + selected: 1, + stream: 0, + invalidated: true + }) + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual(full) + expect(await f.db('snapshot_journal_invalid')).toEqual([{ id: 1, reason: 'capacity-exhausted' }]) + expect(Number((await f.db('snapshot_journal_physical').count('* AS n').first())!.n)).toBe(300) + expect(Number((await f.db('transactions').count('* AS n').first())!.n)).toBe(301) + await expect(copySnapshotJournalBootstrapPage(f.k, 301)).rejects.toThrow('Invalid snapshot') + expect((await copySnapshotJournalBootstrapPage(f.k, 300)).selected).toBe(0) + } finally { + await f.close() + } +}) + +test('MySQL bootstrap refuses a whole over-budget page before allocation or metadata writes', async () => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update({ rowLimit: null }) + await f.db('transactions').insert([sample(0, 1), sample(0, 2)]) + const sequence = f.state.next + expect(await copySnapshotJournalBootstrapPage(f.k, 1)).toEqual({ + complete: false, + selected: 2, + stream: 0, + invalidated: true + }) + expect(f.state.next).toBe(sequence) + expect(await f.db('snapshot_journal_physical')).toEqual([]) + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: null, + rowLimit: 1, + rowsUsed: 0 + }) + expect(await f.db('transactions')).toHaveLength(2) + } finally { + await f.close() + } +}) + +test('MySQL bootstrap rolls back first allowance binding and charge with failed metadata', async () => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update({ rowLimit: null }) + await f.db('transactions').insert(sample(0, 1)) + f.state.failMetadata = true + await expect(copySnapshotJournalBootstrapPage(f.k, 1)).rejects.toThrow('metadata write failed') + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 0, + cursor: null, + rowLimit: null, + rowsUsed: 0 + }) + expect(await f.db('snapshot_journal_physical')).toEqual([]) + f.state.failMetadata = false + await copySnapshotJournalBootstrapPage(f.k, 1) + expect((await f.db('snapshot_journal_bootstrap').first()).rowsUsed).toBe(1) + await copySnapshotJournalBootstrapPage(f.k, 1) + expect((await f.db('snapshot_journal_bootstrap').first()).rowsUsed).toBe(1) + } finally { + await f.close() + } +}) + +test('MySQL bootstrap charges examined records even when live metadata already owns the key', async () => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update({ rowLimit: null }) + await f.db('transactions').insert(sample(0, 1)) + const live = { tableId: 0, id1: 1, id2: 0, exactText: Buffer.from(''), revision: 9, generation: 7, present: 0 } + await f.db('snapshot_journal_physical').insert(live) + await copySnapshotJournalBootstrapPage(f.k, 1) + expect(await f.db('snapshot_journal_physical').first()).toEqual(live) + expect((await f.db('snapshot_journal_bootstrap').first()).rowsUsed).toBe(1) + } finally { + await f.close() + } +}) + +test.each([0, 2147483647])('MySQL bootstrap persists boundary allowance %s for an empty stream', async allowance => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update({ rowLimit: null }) + expect(await copySnapshotJournalBootstrapPage(f.k, allowance)).toEqual({ + complete: false, + selected: 0, + stream: 0, + invalidated: false + }) + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual({ + id: 1, + stream: 1, + cursor: null, + rowLimit: allowance, + rowsUsed: 0 + }) + } finally { + await f.close() + } +}) + +test.each([-1, 0.5, 2147483648, NaN, Infinity, '1', undefined, null])( + 'MySQL bootstrap rejects invalid allowance %s before I/O', + async allowance => { + const f = await driver() + try { + await expect(copySnapshotJournalBootstrapPage(f.k, allowance as number)).rejects.toThrow('Invalid snapshot') + expect(f.queries).toEqual([]) + } finally { + await f.close() + } + } +) + +test.each([{ rowsUsed: 0.5 }, { rowLimit: 0.5 }, { rowLimit: null, rowsUsed: 1 }])( + 'MySQL bootstrap refuses malformed persisted budget %j', + async patch => { + const f = await driver() + try { + await f.db('snapshot_journal_bootstrap').update(patch) + const before = await f.db('snapshot_journal_bootstrap').first() + await expect(copySnapshotJournalBootstrapPage(f.k, 1000000)).rejects.toThrow('Invalid snapshot') + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual(before) + expect(await f.db('snapshot_journal_physical')).toEqual([]) + } finally { + await f.close() + } + } +) + +test('MySQL bootstrap rejects caller transactions before a nested transaction or progress read', async () => { + const f = await driver() + try { + const before = await f.db('snapshot_journal_bootstrap').first() + await f.k.transaction(async t => { + const count = f.queries.length + await expect(copySnapshotJournalBootstrapPage(t, 1000000)).rejects.toThrow('Invalid snapshot') + expect(f.queries).toHaveLength(count) + }) + expect(await f.db('snapshot_journal_bootstrap').first()).toEqual(before) + } finally { + await f.close() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlClock.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlClock.test.ts new file mode 100644 index 000000000..686c0d68d --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlClock.test.ts @@ -0,0 +1,84 @@ +import { knex } from 'knex' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { installSnapshotJournalMysqlClock } from './SnapshotJournalMysqlClock' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' + +test('MySQL clock installation refuses SQLite before issuing SQL', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + const query = jest.fn() + k.on('query', query) + try { + await expect(installSnapshotJournalMysqlClock(k, snapshotJournalRevision('1'))).rejects.toThrow( + 'Snapshot journal clock requires MySQL' + ) + expect(query).not.toHaveBeenCalled() + } finally { + await k.destroy() + } +}) + +test.each(['0', '-1', '01', '9223372036854775808'])( + 'MySQL clock refuses invalid event ceiling %s before DDL or acquisition', + async ceiling => { + const k = knex({ client: 'mysql2' }) + const query = jest.fn() + k.on('query', query) + try { + await expect(installSnapshotJournalMysqlClock(k, ceiling as SnapshotJournalRevision)).rejects.toThrow( + ceiling === '0' ? 'Empty snapshot journal event window' : 'Invalid snapshot journal revision' + ) + expect(query).not.toHaveBeenCalled() + } finally { + await k.destroy() + } + } +) + +test.each(['mysql', 'mysql2'])( + 'MySQL clock alias %s creates its native objects and binds its exact ceiling', + async client => { + const k = knex({ client: 'mysql2' }) + k.client.config.client = client + const captured: { ddl: Array<{ sql: string; bindings: unknown[] }> } = JSON.parse( + readFileSync( + join(__dirname, '../../../../test/fixtures/snapshotJournal/mysql-generation-ddl-fixture.json'), + 'utf8' + ) + ) + const expected = captured.ddl.filter(row => /^CREATE TABLE snapshot_journal_(clock|events|invalid)\(/.test(row.sql)) + expect(expected).toHaveLength(3) + const queries: Array<{ sql: string; bindings: unknown[] }> = [] + const connection = { + query( + query: { sql: string }, + bindings: unknown[], + callback: (error: Error | null, rows?: unknown[], fields?: unknown[]) => void + ) { + queries.push({ sql: query.sql, bindings }) + callback(null, [], []) + } + } + jest.spyOn(k.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(k.client, 'releaseConnection').mockResolvedValue(undefined) + try { + await installSnapshotJournalMysqlClock(k, snapshotJournalRevision('9223372036854775807')) + // The generation owner adds its storage options, epoch and atomic initial row. + // This primitive emits the same base table shapes, then binds the clock row. + expect(queries.slice(0, 3)).toEqual( + expected.map(row => ({ sql: row.sql.split(' DEFAULT CHARACTER SET ')[0], bindings: [] })) + ) + expect(queries[3]).toEqual({ + sql: 'insert into `snapshot_journal_clock` (`ceiling`, `id`) values (?, ?)', + bindings: ['9223372036854775807', 1] + }) + expect(queries).toHaveLength(4) + } finally { + await k.destroy() + } + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlClock.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlClock.ts new file mode 100644 index 000000000..721fbe641 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlClock.ts @@ -0,0 +1,43 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { + MAX_SNAPSHOT_JOURNAL_REVISION, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' + +export const SNAPSHOT_JOURNAL_MYSQL_CLOCK_DDL = [ + 'CREATE TABLE snapshot_journal_clock(id INTEGER NOT NULL PRIMARY KEY,ceiling BIGINT UNSIGNED NOT NULL,CHECK(id=1)) ENGINE=InnoDB', + 'CREATE TABLE snapshot_journal_events(revision BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY) ENGINE=InnoDB', + "CREATE TABLE snapshot_journal_invalid(id INTEGER NOT NULL PRIMARY KEY,reason VARCHAR(32) NOT NULL,CHECK(id=1),CHECK(reason IN ('capacity-exhausted','revision-exhausted','key-out-of-range'))) ENGINE=InnoDB" +] + +export const SNAPSHOT_JOURNAL_MYSQL_CLOCK_VARIABLES = `DECLARE journalCeiling BIGINT UNSIGNED; +DECLARE journalRevision BIGINT UNSIGNED; +DECLARE journalEnabled BOOLEAN; ` + +/** Shared barrier; invalidation uses another row. Unexpected native errors roll back the source transaction. */ +export const SNAPSHOT_JOURNAL_MYSQL_ADVANCE = `SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE; +SELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled; +IF journalEnabled THEN + INSERT INTO snapshot_journal_events() VALUES(); + SET journalRevision=LAST_INSERT_ID(); + DELETE FROM snapshot_journal_events WHERE revision=journalRevision; + IF journalRevision>journalCeiling THEN + INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>${MAX_SNAPSHOT_JOURNAL_REVISION},'revision-exhausted','capacity-exhausted')); + SET journalEnabled=FALSE; + END IF; +END IF; ` + +/** Fresh owned fixture/installation only; registered partial-DDL recovery is separate. */ +export async function installSnapshotJournalMysqlClock(k: Knex, ceiling: SnapshotJournalRevision): Promise { + if (k.client.config.client !== 'mysql' && k.client.config.client !== 'mysql2') { + throw new WERR_INVALID_OPERATION('Snapshot journal clock requires MySQL') + } + if (snapshotJournalRevision(ceiling) === '0') throw new WERR_INVALID_OPERATION('Empty snapshot journal event window') + await runInSeries(SNAPSHOT_JOURNAL_MYSQL_CLOCK_DDL, async statement => { + await k.raw(statement) + }) + await k('snapshot_journal_clock').insert({ id: 1, ceiling }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.test.ts new file mode 100644 index 000000000..a2b0c5c23 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.test.ts @@ -0,0 +1,932 @@ +import * as observers from './SnapshotJournalMysqlObservers' +import { knex, type Knex } from 'knex' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { + completeSnapshotJournalMysqlGeneration, + installSnapshotJournalMysqlGeneration, + readSnapshotJournalMysqlGeneration +} from './SnapshotJournalMysqlGeneration' +import { readSnapshotJournalMysqlBinding } from './SnapshotJournalMysqlSource' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' + +const journalReceiptPolicy = { receiptLimit: 128, receiptLifetimeMs: 2592000000 } + +jest.mock('./SnapshotJournalMysqlSource', () => ({ readSnapshotJournalMysqlBinding: jest.fn() })) +interface Capture { + sql: string + bindings?: unknown[] + rows: Array> +} +const captured: Capture[] = JSON.parse( + readFileSync( + join(__dirname, '../../../../test/fixtures/snapshotJournal/mysql-generation-metadata-fixture.json'), + 'utf8' + ) +) +const nativeState = JSON.parse( + readFileSync(join(__dirname, '../../../../test/fixtures/snapshotJournal/mysql-generation-state-fixture.json'), 'utf8') +) as Record +let ceiling: SnapshotJournalRevision +beforeEach(() => { + ceiling = snapshotJournalRevision('9223372036854775807') +}) +const readBinding = jest.mocked(readSnapshotJournalMysqlBinding) +const nativeDdl: { epoch: string; ddl: Array<{ sql: string; bindings: unknown[] }> } = JSON.parse( + readFileSync(join(__dirname, '../../../../test/fixtures/snapshotJournal/mysql-generation-ddl-fixture.json'), 'utf8') +) +async function fixture() { + const database = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + await database.raw( + 'CREATE TABLE snapshot_journal_generation(id INTEGER,version INTEGER,epoch TEXT,source TEXT,plan TEXT,ceiling TEXT,nextObject INTEGER,complete INTEGER)' + ) + await database('snapshot_journal_generation').insert(nativeState) + await database.raw('CREATE TABLE snapshot_journal_clock(id INTEGER,ceiling TEXT)') + await database('snapshot_journal_clock').insert({ id: 1, ceiling }) + await database.raw( + 'CREATE TABLE snapshot_journal_bootstrap(id INTEGER,stream INTEGER,cursor TEXT,rowLimit INTEGER,rowsUsed INTEGER)' + ) + await database('snapshot_journal_bootstrap').insert({ + id: 1, + stream: 17, + cursor: null, + rowLimit: 1000000, + rowsUsed: 0 + }) + await database.raw('CREATE TABLE snapshot_journal_invalid(id INTEGER,reason TEXT)') + await database.raw('CREATE TABLE snapshot_journal_events(revision TEXT)') + await database.raw( + 'CREATE TABLE snapshot_journal_retention(id INTEGER,floor TEXT,receiptLimit INTEGER,receiptLifetimeMs BIGINT)' + ) + await database('snapshot_journal_retention').insert({ id: 1, floor: '0', ...journalReceiptPolicy }) + await database.raw( + 'CREATE TABLE snapshot_journal_receipts(requestId TEXT,binding TEXT,highWater TEXT,floor TEXT,expiresAt BIGINT)' + ) + const metadata = structuredClone(captured), + writes: string[] = [] + database.on('query', query => { + if (/^(?:insert|update|delete|create|alter|drop)/i.test(query.sql)) writes.push(query.sql) + }) + readBinding.mockResolvedValue(String(nativeState.source)) + let activeEpoch = String(nativeState.epoch), + available: Set | undefined + const fault = { + phase: '', + fired: false, + zeroNext: false, + zeroComplete: false, + beforeTransaction: undefined as undefined | ((t: Knex) => Promise) + } + const fail = (phase: string): void => { + if (fault.phase === phase && !fault.fired) { + fault.fired = true + throw new Error('lost reply at ' + phase) + } + } + const raw = jest.fn((sql: string, values?: unknown[]) => { + if ( + [ + 'CAST(ceiling AS CHAR) ceiling', + 'CAST(receiptLifetimeMs AS CHAR) receiptLifetimeMs', + 'SUBSTRING(floor,1,20) floor', + '1 AS occupied' + ].includes(sql) + ) + return database.raw(sql) + if (sql === 'SELECT VERSION() version') return Promise.resolve([[{ version: '8.4.0' }]]) + const name = /^CREATE (?:TABLE|TRIGGER) (snapshot_journal_\w+)/.exec(sql)?.[1] + if (name) + return (async () => { + fail('before:' + name) + const reference = nativeDdl.ddl.find( + entry => /^CREATE (?:TABLE|TRIGGER) (snapshot_journal_\w+)/.exec(entry.sql)?.[1] === name + ) + if (!reference || !available || available.has(name)) throw new Error('Unowned fixture DDL') + if (name === 'snapshot_journal_generation') activeEpoch = String(values?.[0]) + expect(sql).toBe(reference.sql.replaceAll(nativeDdl.epoch, activeEpoch)) + expect(values ?? []).toEqual( + name === 'snapshot_journal_generation' + ? [activeEpoch, nativeState.source, nativeState.plan, ceiling] + : reference.bindings + ) + if (name === 'snapshot_journal_generation') + await database(name).insert({ + id: 1, + version: 1, + epoch: activeEpoch, + source: values![1], + plan: values![2], + ceiling: values![3], + nextObject: 0, + complete: 0 + }) + if (name === 'snapshot_journal_clock') await database(name).insert({ id: 1, ceiling: values![0] }) + if (name === 'snapshot_journal_bootstrap') + await database(name).insert({ id: 1, stream: 0, cursor: null, rowLimit: null, rowsUsed: 0 }) + if (name === 'snapshot_journal_retention') + await database(name).insert({ id: 1, floor: '0', receiptLimit: values![0], receiptLifetimeMs: values![1] }) + available.add(name) + fail('after:' + name) + return {} + })() + const found = metadata.find( + entry => entry.sql === sql && JSON.stringify(entry.bindings ?? []) === JSON.stringify(values ?? []) + ) + if (!found) throw new Error('Unexpected native metadata request: ' + sql) + let result = JSON.parse(JSON.stringify(found.rows).replaceAll(String(nativeState.epoch), activeEpoch)) as Array< + Record + > + if (available) { + if (sql.includes('LOWER(LEFT(')) result = result.filter(row => available!.has(String(row.name))) + else { + const owned = values?.find(value => typeof value === 'string' && value.startsWith('snapshot_journal_')) + if (typeof owned === 'string' && !available.has(owned)) result = [] + } + } + return Promise.resolve([result]) + }) + const wrap = (connection: Knex, isTransaction = false): Knex => + Object.assign( + (table: string) => { + const builder = connection(table) + // Only the native fixture establishes MySQL row-lock behavior. This fixture + // executes state DML/rollback in SQLite and owns exact native DDL/metadata. + builder.forUpdate = () => builder + builder.noWait = () => builder + const update = builder.update.bind(builder) + builder.update = ((patch: Record) => + (async () => { + if ('nextObject' in patch && fault.zeroNext) return 0 + if ('complete' in patch && fault.zeroComplete) return 0 + if ('complete' in patch) fail('complete:before') + const changed = await update(patch) + if ('nextObject' in patch) fail('ack:' + String(patch.nextObject)) + if ('complete' in patch) fail('complete:after') + return changed + })()) as typeof builder.update + return builder + }, + { + client: { config: { client: 'mysql2' } }, + isTransaction, + raw, + transaction: async (callback: (t: Knex) => Promise) => + await connection.transaction(async t => { + await fault.beforeTransaction?.(t) + return await callback(wrap(t, true)) + }) + } + ) as unknown as Knex + const k = wrap(database) + const fresh = async (): Promise => { + for (const table of [ + 'snapshot_journal_generation', + 'snapshot_journal_clock', + 'snapshot_journal_bootstrap', + 'snapshot_journal_invalid', + 'snapshot_journal_events', + 'snapshot_journal_retention', + 'snapshot_journal_receipts' + ]) + await database(table).delete() + available = new Set() + writes.length = 0 + } + const rows = (fragment: string, table?: string): Array> => { + const entry = metadata.find( + entry => entry.sql.includes(fragment) && (table === undefined || entry.bindings?.includes(table)) + ) + if (!entry) throw new Error('Missing native fixture metadata: ' + fragment) + return entry.rows + } + return { database, k, metadata, rows, writes, raw, fresh, fault } +} +test('native metadata resumes the complete persisted generation without writes', async () => { + const f = await fixture() + try { + expect(await readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toEqual({ + source: nativeState.source, + plan: nativeState.plan, + ceiling, + epoch: nativeState.epoch, + nextObject: 59, + complete: true, + enabled: true + }) + expect(await installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toEqual( + await readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy) + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) +test('lost final object acknowledgement resumes its epoch and advances only the durable intent', async () => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ nextObject: 58, complete: 0 }) + await f.database('snapshot_journal_bootstrap').update({ stream: 0, rowLimit: null }) + f.writes.length = 0 + const resumed = await installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy) + expect(resumed).toMatchObject({ + epoch: nativeState.epoch, + nextObject: 59, + complete: false, + enabled: true + }) + expect(f.writes).toHaveLength(1) + expect(f.writes[0]).toMatch(/^update `snapshot_journal_generation`/) + } finally { + await f.database.destroy() + } +}) +test.each(['sqlite3', 'better-sqlite3', 'pg'])('unsupported driver %s performs no I/O', async client => { + const f = await fixture() + try { + f.k.client.config.client = client + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.raw).not.toHaveBeenCalled() + } finally { + await f.database.destroy() + } +}) +test('DDL refuses a caller-owned transaction', async () => { + const f = await fixture() + try { + Object.defineProperty(f.k, 'isTransaction', { value: true }) + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.raw).not.toHaveBeenCalled() + } finally { + await f.database.destroy() + } +}) +test('generation reads accept the caller pinned view without DDL', async () => { + const f = await fixture() + try { + Object.defineProperty(f.k, 'isTransaction', { value: true }) + expect(await readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toMatchObject({ + epoch: nativeState.epoch, + complete: true, + enabled: true + }) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) +test('persisted allocator rows cannot establish event continuity', async () => { + const f = await fixture() + try { + await f.database('snapshot_journal_events').insert({ revision: '1' }) + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } +}) +test.each(['0', '01', '9223372036854775808'])('invalid ceiling %s refuses before metadata', async value => { + const f = await fixture() + try { + await expect( + installSnapshotJournalMysqlGeneration(f.k, value as typeof ceiling, journalReceiptPolicy) + ).rejects.toThrow() + expect(f.raw).not.toHaveBeenCalled() + } finally { + await f.database.destroy() + } +}) +test.each([ + ['TABLE_COMMENT comment', 'engine', 'MyISAM'], + ['TABLE_COMMENT comment', 'type', 'VIEW'], + ['TABLE_COMMENT comment', 'collation', 'utf8mb4_general_ci'], + ['TABLE_COMMENT comment', 'rowFormat', 'Compact'], + ['TABLE_COMMENT comment', 'options', ''], + ['TABLE_COMMENT comment', 'comment', 'snapshot-journal-owner:foreign'], + ['ORDINAL_POSITION LIMIT 9', 'name', 'foreign'], + ['ORDINAL_POSITION LIMIT 9', 'type', 'bigint'], + ['ORDINAL_POSITION LIMIT 9', 'nullable', 'YES'], + ['ORDINAL_POSITION LIMIT 9', 'defaultValue', 0], + ['ORDINAL_POSITION LIMIT 9', 'extra', 'auto_increment'], + ['ORDINAL_POSITION LIMIT 9', 'charset', 'utf8mb4'], + ['ORDINAL_POSITION LIMIT 9', 'collation', 'utf8mb4_bin'], + ['ORDINAL_POSITION LIMIT 9', 'expression', '1'], + ['SEQ_IN_INDEX LIMIT 16', 'name', 'foreign'], + ['SEQ_IN_INDEX LIMIT 16', 'columnName', 'ceiling'], + ['SEQ_IN_INDEX LIMIT 16', 'position', 2], + ['SEQ_IN_INDEX LIMIT 16', 'nonUnique', 1], + ['SEQ_IN_INDEX LIMIT 16', 'direction', 'D'], + ['SEQ_IN_INDEX LIMIT 16', 'prefix', 1], + ['SEQ_IN_INDEX LIMIT 16', 'type', 'HASH'], + ['SEQ_IN_INDEX LIMIT 16', 'visible', 'NO'], + ['SEQ_IN_INDEX LIMIT 16', 'expression', '1'], + ['CONSTRAINT_NAME LIMIT 8', 'name', 'foreign'], + ['CONSTRAINT_NAME LIMIT 8', 'type', 'CHECK'], + ['CONSTRAINT_NAME LIMIT 8', 'enforced', 'NO'], + ['CONSTRAINT_NAME LIMIT 8', 'clause', '(`id` = 2)'] +])('table metadata %s %s drift refuses before mutation', async (fragment, field, value) => { + const f = await fixture() + try { + f.rows(fragment as string, 'snapshot_journal_clock')[0][field as string] = value + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) +test.each(['TABLE_COMMENT comment', 'ORDINAL_POSITION LIMIT 9', 'SEQ_IN_INDEX LIMIT 16', 'CONSTRAINT_NAME LIMIT 8'])( + 'missing %s metadata refuses', + async fragment => { + const f = await fixture() + try { + f.rows(fragment, 'snapshot_journal_clock').pop() + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } + } +) +test.each(['EVENT_OBJECT_TABLE=? LIMIT 1', 'PARTITION_NAME IS NOT NULL LIMIT 1', 'REFERENCED_TABLE_NAME=? LIMIT 1'])( + 'unowned observer/partition/foreign dependency %s refuses', + async fragment => { + const f = await fixture() + try { + f.rows(fragment, 'snapshot_journal_clock').push({ name: 'foreign' }) + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } + } +) +test.each([ + ['name', 'foreign'], + ['table', 'transactions'], + ['event', 'DELETE'], + ['timing', 'BEFORE'], + ['body', 'BEGIN DO 0; END'], + ['sqlMode', ''], + ['charset', 'ascii'], + ['collation', 'ascii_bin'], + ['databaseCollation', 'ascii_bin'], + ['definer', 'foreign@localhost'] +])('trigger %s drift refuses exact ownership', async (field, value) => { + const f = await fixture() + try { + f.rows('SUBSTRING(ACTION_STATEMENT,1,?)', 'snapshot_journal_scope_0_INSERT')[0][field] = value + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } +}) +test.each(['unknown', 'view', 'missing-prior', 'duplicate', 'future', 'many'])( + 'reserved object %s cannot establish continuity', + async kind => { + const f = await fixture() + try { + const rows = f.rows('LOWER(LEFT(TABLE_NAME,17))') + if (kind === 'unknown') rows.push({ name: 'snapshot_journal_foreign', type: 'BASE TABLE' }) + if (kind === 'view') rows.find(row => row.name === 'snapshot_journal_clock')!.type = 'VIEW' + if (kind === 'missing-prior') + rows.splice( + rows.findIndex(row => row.name === 'snapshot_journal_clock'), + 1 + ) + if (kind === 'duplicate') rows.push({ name: 'snapshot_journal_clock', type: 'BASE TABLE' }) + if (kind === 'future') await f.database('snapshot_journal_generation').update({ nextObject: 0, complete: 0 }) + if (kind === 'many') + rows.push( + ...Array.from({ length: 130 }, (_, i) => ({ + name: 'snapshot_journal_foreign_' + i, + type: 'BASE TABLE' + })) + ) + f.writes.length = 0 + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } + } +) +test.each([ + { nextObject: 60 }, + { nextObject: 58 }, + { source: 'a'.repeat(64) }, + { plan: 'a'.repeat(64) }, + { ceiling: '1' } +])('intent drift %j refuses', async patch => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update(patch) + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } +}) +test.each([ + { id: 2 }, + { stream: -1 }, + { stream: 18 }, + { stream: 0.5 }, + { stream: 16 }, + { cursor: '[]' }, + { cursor: 'x'.repeat(2049) } +])('completed bootstrap drift %j refuses', async patch => { + const f = await fixture() + try { + await f.database('snapshot_journal_bootstrap').update(patch) + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } +}) +test.each(['snapshot_journal_clock', 'snapshot_journal_bootstrap'])( + 'missing %s seed cannot resume or publish', + async table => { + const f = await fixture() + try { + await f.database(table).delete() + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } + } +) +test.each(['capacity-exhausted', 'revision-exhausted', 'key-out-of-range'])( + 'valid invalidation %s reports disabled while preserving source state', + async reason => { + const f = await fixture() + try { + await f.database('snapshot_journal_invalid').insert({ id: 1, reason }) + expect(await readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toMatchObject({ + complete: true, + enabled: false + }) + } finally { + await f.database.destroy() + } + } +) +test.each([ + { id: 2, reason: 'capacity-exhausted' }, + { id: 1, reason: 'unknown' } +])('malformed invalidation %j refuses', async row => { + const f = await fixture() + try { + await f.database('snapshot_journal_invalid').insert(row) + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await f.database.destroy() + } +}) + +const objectNames = nativeDdl.ddl.map(entry => /^CREATE (?:TABLE|TRIGGER) (snapshot_journal_\w+)/.exec(entry.sql)![1]) +test('fresh installation uses the independently captured native DDL and atomically seeded controls', async () => { + const f = await fixture() + try { + await f.fresh() + const state = await installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy) + expect(state).toMatchObject({ nextObject: 59, complete: false, enabled: true }) + expect(state.epoch).not.toBe(nativeState.epoch) + expect(await f.database('snapshot_journal_clock')).toEqual([{ id: 1, ceiling }]) + expect(await f.database('snapshot_journal_bootstrap')).toEqual([ + { id: 1, stream: 0, cursor: null, rowLimit: null, rowsUsed: 0 } + ]) + expect(objectNames).toHaveLength(60) + } finally { + await f.database.destroy() + } +}) +test.each(objectNames)('every DDL acknowledgement loss resumes without adopting/replacing %s', async name => { + const f = await fixture() + try { + await f.fresh() + f.fault.phase = 'after:' + name + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'lost reply' + ) + const before = await f.database('snapshot_journal_generation').first() + const resumed = await installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy) + expect(f.fault.fired).toBe(true) + expect(resumed).toMatchObject({ + epoch: before.epoch, + nextObject: 59, + complete: false, + enabled: true + }) + expect( + f.raw.mock.calls.filter(([sql]) => /^CREATE (?:TABLE|TRIGGER) (snapshot_journal_\w+)/.exec(sql)?.[1] === name) + ).toHaveLength(1) + } finally { + await f.database.destroy() + } +}) +test.each([0, 1, 5, 6, 7, 8, 59])('before DDL boundary %i creates no undocumented object', async index => { + const f = await fixture() + try { + await f.fresh() + f.fault.phase = 'before:' + objectNames[index] + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'lost reply' + ) + expect(await installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toMatchObject({ + nextObject: 59, + complete: false, + enabled: true + }) + } finally { + await f.database.destroy() + } +}) +test.each([1, 6, 7, 8, 9, 59])('lost progress acknowledgement %i resumes committed state', async position => { + const f = await fixture() + try { + await f.fresh() + f.fault.phase = 'ack:' + position + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'lost reply' + ) + expect((await f.database('snapshot_journal_generation').first()).nextObject).toBe(position) + expect(await installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toMatchObject({ + nextObject: 59, + complete: false, + enabled: true + }) + } finally { + await f.database.destroy() + } +}) +test('completion commits once and can be read in the same retained generation', async () => { + const f = await fixture() + try { + expect(await completeSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toMatchObject({ + epoch: nativeState.epoch, + nextObject: 59, + complete: true, + enabled: true + }) + expect((await f.database('snapshot_journal_generation').first()).complete).toBe(1) + } finally { + await f.database.destroy() + } +}) +test.each(['complete:before', 'complete:after'])( + 'completion %s loss rolls back its durable publication', + async phase => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ complete: 0 }) + f.fault.phase = phase + await expect(completeSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'lost reply' + ) + expect((await f.database('snapshot_journal_generation').first()).complete).toBe(0) + expect(await completeSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toMatchObject({ + complete: true, + enabled: true + }) + } finally { + await f.database.destroy() + } + } +) +test.each(['incomplete', 'invalidated'])('completion refuses %s progress without publication', async kind => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ complete: 0 }) + if (kind === 'incomplete') await f.database('snapshot_journal_bootstrap').update({ stream: 16 }) + else await f.database('snapshot_journal_invalid').insert({ id: 1, reason: 'capacity-exhausted' }) + await expect(completeSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect((await f.database('snapshot_journal_generation').first()).complete).toBe(0) + } finally { + await f.database.destroy() + } +}) + +// Context is part of persisted trigger ownership and the plan digest. A malformed +// driver response must not establish a weaker ownership contract. +test.each(['missing row', 'extra row', 'missing field', 'extra field', 'numeric field', 'oversized field'])( + 'installation rejects malformed native context: %s', + async kind => { + const f = await fixture() + try { + const rows = f.rows('SELECT @@sql_mode') + if (kind === 'missing row') rows.length = 0 + if (kind === 'extra row') rows.push({ ...rows[0] }) + if (kind === 'missing field') delete rows[0].definer + if (kind === 'extra field') rows[0].unexpected = 'unknown' + if (kind === 'numeric field') rows[0].charset = 1 + if (kind === 'oversized field') rows[0].sqlMode = 'x'.repeat(4097) + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } + } +) +test.each(['malformed trigger', 'missing trigger', 'duplicate trigger'])( + 'installation rejects an inconsistent generated plan: %s', + async kind => { + const f = await fixture() + let spy: jest.SpyInstance | undefined + try { + const definitions = await observers.snapshotJournalMysqlObserverSql(f.k) + if (kind === 'malformed trigger') definitions[0] = 'CREATE TABLE unrelated(id INT)' + if (kind === 'missing trigger') definitions.pop() + if (kind === 'duplicate trigger') definitions[1] = definitions[0] + spy = jest.spyOn(observers, 'snapshotJournalMysqlObserverSql').mockResolvedValue(definitions) + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + spy?.mockRestore() + await f.database.destroy() + } + } +) +test('zero journal capacity refuses before metadata reads or writes', async () => { + const f = await fixture() + try { + await expect( + installSnapshotJournalMysqlGeneration(f.k, snapshotJournalRevision('0'), journalReceiptPolicy) + ).rejects.toThrow('Invalid or unowned') + expect(f.raw).not.toHaveBeenCalled() + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) + +test('installation refuses a changed source binding before returning generation ownership', async () => { + const f = await fixture() + try { + readBinding.mockResolvedValueOnce(String(nativeState.source)).mockResolvedValueOnce('f'.repeat(64)) + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) +test('lost update ownership cannot acknowledge an installation object', async () => { + const f = await fixture() + try { + await f.fresh() + f.fault.zeroNext = true + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect((await f.database('snapshot_journal_generation').first()).nextObject).toBe(0) + f.fault.zeroNext = false + expect(await installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).toMatchObject({ + nextObject: 59, + complete: false + }) + } finally { + await f.database.destroy() + } +}) +test('lost update ownership cannot publish completion', async () => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ complete: 0 }) + f.fault.zeroComplete = true + await expect(completeSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect((await f.database('snapshot_journal_generation').first()).complete).toBe(0) + } finally { + await f.database.destroy() + } +}) +test.each(['clock missing', 'epoch changed', 'progress changed', 'bootstrap missing', 'invalidated'])( + 'publication revalidates transaction-owned %s and rolls back refusal', + async kind => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ complete: 0 }) + f.fault.beforeTransaction = async t => { + if (kind === 'clock missing') await t('snapshot_journal_clock').delete() + if (kind === 'epoch changed') + await t('snapshot_journal_generation').update({ + epoch: 'aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa' + }) + if (kind === 'progress changed') await t('snapshot_journal_generation').update({ nextObject: 58 }) + if (kind === 'bootstrap missing') await t('snapshot_journal_bootstrap').delete() + if (kind === 'invalidated') await t('snapshot_journal_invalid').insert({ id: 1, reason: 'capacity-exhausted' }) + } + await expect(completeSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect((await f.database('snapshot_journal_generation').first()).complete).toBe(0) + expect(await f.database('snapshot_journal_clock')).toEqual([{ id: 1, ceiling }]) + expect(await f.database('snapshot_journal_invalid')).toEqual([]) + } finally { + await f.database.destroy() + } + } +) +test('installation refuses a partial bootstrap cursor before resuming DDL', async () => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ nextObject: 58, complete: 0 }) + await f.database('snapshot_journal_bootstrap').update({ stream: 1 }) + f.writes.length = 0 + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) +test.each([60, 58])('installation refuses invalid complete next-object position %i', async nextObject => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ nextObject }) + f.writes.length = 0 + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) +test('generation requires its intent in the reserved-object inventory', async () => { + const f = await fixture() + try { + const rows = f.rows('LOWER(LEFT(TABLE_NAME,17))') + rows.splice( + rows.findIndex(row => row.name === 'snapshot_journal_generation'), + 1 + ) + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) + +test('final state rechecks a clock response after object validation', async () => { + const f = await fixture() + let reads = 0 + const listener = (rows: unknown[], query: { sql: string }) => { + if (query.sql.startsWith('select `id`, CAST(ceiling AS CHAR) ceiling from `snapshot_journal_clock`')) { + reads++ + if (reads === 2) rows.splice(0) + } + } + f.database.on('query-response', listener) + try { + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(reads).toBe(2) + } finally { + f.database.off('query-response', listener) + await f.database.destroy() + } +}) + +test('receipt policy is immutable across installation, reads and completion', async () => { + const f = await fixture() + try { + for (const policy of [ + { ...journalReceiptPolicy, receiptLimit: 127 }, + { ...journalReceiptPolicy, receiptLifetimeMs: 1000 } + ]) { + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, policy)).rejects.toThrow() + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, policy)).rejects.toThrow() + await expect(completeSnapshotJournalMysqlGeneration(f.k, ceiling, policy)).rejects.toThrow() + } + expect(f.writes).toEqual([]) + expect(await f.database('snapshot_journal_retention')).toEqual([{ id: 1, floor: '0', ...journalReceiptPolicy }]) + } finally { + await f.database.destroy() + } +}) + +test('opening snapshots the policy before an asynchronous source read', async () => { + const f = await fixture() + try { + const policy = { ...journalReceiptPolicy } + readBinding.mockImplementationOnce(async () => { + policy.receiptLimit = 1 + policy.receiptLifetimeMs = 1 + return String(nativeState.source) + }) + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, policy)).resolves.toMatchObject({ complete: true }) + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) + +test.each([ + null, + undefined, + [], + {}, + { receiptLimit: 0, receiptLifetimeMs: 1 }, + { receiptLimit: 129, receiptLifetimeMs: 1 }, + { receiptLimit: 1.5, receiptLifetimeMs: 1 }, + { receiptLimit: 1, receiptLifetimeMs: 0 }, + { receiptLimit: 1, receiptLifetimeMs: 2592000001 } +])('invalid receipt policy %o refuses before SQL', async policy => { + const f = await fixture() + try { + f.raw.mockClear() + await expect( + installSnapshotJournalMysqlGeneration(f.k, ceiling, policy as typeof journalReceiptPolicy) + ).rejects.toThrow() + await expect( + readSnapshotJournalMysqlGeneration(f.k, ceiling, policy as typeof journalReceiptPolicy) + ).rejects.toThrow() + await expect( + completeSnapshotJournalMysqlGeneration(f.k, ceiling, policy as typeof journalReceiptPolicy) + ).rejects.toThrow() + expect(f.raw).not.toHaveBeenCalled() + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) + +test.each(['missing', 'duplicate', 'limit', 'lifetime', 'floor', 'capacity'])( + 'refuses invalid retained receipt state: %s', + async kind => { + const f = await fixture() + try { + if (kind === 'missing') await f.database('snapshot_journal_retention').delete() + if (kind === 'duplicate') + await f.database('snapshot_journal_retention').insert({ id: 2, floor: '0', ...journalReceiptPolicy }) + if (kind === 'limit') await f.database('snapshot_journal_retention').update({ receiptLimit: 127 }) + if (kind === 'lifetime') await f.database('snapshot_journal_retention').update({ receiptLifetimeMs: 1000 }) + if (kind === 'floor') await f.database('snapshot_journal_retention').update({ floor: '01' }) + if (kind === 'capacity') + await f.database('snapshot_journal_receipts').insert( + Array.from({ length: 129 }, (_, n) => ({ + requestId: String(n), + binding: 'b', + highWater: '1', + floor: '0', + expiresAt: 1 + })) + ) + f.writes.length = 0 + await expect(readSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow() + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } + } +) + +test('partially installed retention cannot claim a progressed floor', async () => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update({ nextObject: 58, complete: 0 }) + await f.database('snapshot_journal_bootstrap').update({ stream: 0, cursor: null, rowLimit: null, rowsUsed: 0 }) + await f.database('snapshot_journal_retention').update({ floor: '1' }) + f.writes.length = 0 + await expect(installSnapshotJournalMysqlGeneration(f.k, ceiling, journalReceiptPolicy)).rejects.toThrow() + expect(f.writes).toEqual([]) + } finally { + await f.database.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.ts new file mode 100644 index 000000000..26ac8e017 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.ts @@ -0,0 +1,605 @@ +import type { Knex } from 'knex' +import { createHash } from 'node:crypto' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { SNAPSHOT_JOURNAL_MYSQL_CLOCK_DDL } from './SnapshotJournalMysqlClock' +import { SNAPSHOT_JOURNAL_MYSQL_METADATA_DDL, snapshotJournalMysqlObserverSql } from './SnapshotJournalMysqlObservers' +import { SNAPSHOT_JOURNAL_BOOTSTRAP_DDL, validSnapshotJournalBootstrapBudget } from './SnapshotJournalBootstrap' +import { readSnapshotJournalMysqlBinding } from './SnapshotJournalMysqlSource' +import { + compareSnapshotJournalRevisions, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' +import { + snapshotJournalReceiptDdl, + snapshotJournalReceiptPolicy, + type SnapshotJournalReceiptPolicy +} from './SnapshotJournalReceipt' +import { + createSnapshotJournalMysqlIntent, + readSnapshotJournalMysqlIntent, + SNAPSHOT_JOURNAL_MYSQL_INTENT, + type SnapshotJournalMysqlIntent, + type SnapshotJournalMysqlBinding +} from './SnapshotJournalMysqlIntent' + +interface Column { + name: string + type: string + nullable?: boolean + auto?: boolean +} +interface Index { + name: string + columns: string[] + unique: boolean +} +interface Table { + name: string + sql: string + columns: Column[] + indexes: Index[] + checks: string[] + seed?: 'clock' | 'bootstrap' | 'retention' + charset?: 'ascii' +} +interface Trigger { + name: string + sql: string + table: string + event: string + body: string +} +type ObjectDefinition = { type: 'table'; definition: Table } | { type: 'trigger'; definition: Trigger } +interface Context { + sqlMode: string + charset: string + collation: string + databaseCollation: string + definer: string +} +interface Plan { + binding: SnapshotJournalMysqlBinding + objects: ObjectDefinition[] + context: Context + receiptPolicy: SnapshotJournalReceiptPolicy +} +export interface SnapshotJournalMysqlGeneration extends SnapshotJournalMysqlIntent { + enabled: boolean +} + +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid or unowned MySQL snapshot journal generation') +} +function client(k: Knex): void { + if (!['mysql', 'mysql2'].includes(k.client.config.client)) invalid() +} +const primary = (...columns: string[]): Index => ({ name: 'PRIMARY', columns, unique: true }) +const column = (name: string, type: string): Column => ({ name, type }) +const integer = (name: string): Column => column(name, 'int') +const big = (name: string): Column => column(name, 'bigint unsigned') +const physicalKey = ['tableId', 'id1', 'id2', 'exactText'] +const physicalColumns = [integer('tableId'), big('id1'), big('id2'), column('exactText', 'varbinary(400)')] +const metadataColumns = [big('revision'), column('present', 'tinyint')] +const tail = ' ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT=DYNAMIC' +const objectCount = 59 + +function tables(k: Knex): Table[] { + const ddl = [ + ...SNAPSHOT_JOURNAL_MYSQL_CLOCK_DDL, + ...SNAPSHOT_JOURNAL_MYSQL_METADATA_DDL, + SNAPSHOT_JOURNAL_BOOTSTRAP_DDL, + ...snapshotJournalReceiptDdl(k) + ] + const specs: Omit[] = [ + { + name: 'snapshot_journal_clock', + columns: [integer('id'), big('ceiling')], + indexes: [primary('id')], + checks: ['(`id` = 1)'], + seed: 'clock' + }, + { + name: 'snapshot_journal_events', + columns: [{ ...big('revision'), auto: true }], + indexes: [primary('revision')], + checks: [] + }, + { + name: 'snapshot_journal_invalid', + columns: [integer('id'), column('reason', 'varchar(32)')], + indexes: [primary('id')], + checks: [ + '(`id` = 1)', + "(`reason` in (_utf8mb4'capacity-exhausted',_utf8mb4'revision-exhausted',_utf8mb4'key-out-of-range'))" + ] + }, + { + name: 'snapshot_journal_physical', + columns: [...physicalColumns, big('revision'), big('generation'), column('present', 'tinyint')], + indexes: [ + primary(...physicalKey), + { + name: 'snapshot_journal_physical_page', + columns: ['tableId', 'revision', 'id1', 'id2', 'exactText'], + unique: false + } + ], + checks: [] + }, + { + name: 'snapshot_journal_scope', + columns: [integer('tableId'), big('userId'), ...physicalColumns.slice(1), ...metadataColumns], + indexes: [ + primary('tableId', 'userId', 'id1', 'id2', 'exactText'), + { + name: 'snapshot_journal_scope_page', + columns: ['userId', 'tableId', 'revision', 'id1', 'id2', 'exactText'], + unique: false + } + ], + checks: [] + }, + { + name: 'snapshot_journal_bootstrap', + columns: [ + integer('id'), + integer('stream'), + { ...column('cursor', 'text'), nullable: true }, + { ...integer('rowLimit'), nullable: true }, + integer('rowsUsed') + ], + indexes: [primary('id')], + checks: [ + '(`id` = 1)', + '(`stream` between 0 and 17)', + '((`rowLimit` is null) or (`rowLimit` between 0 and 2147483647))', + '(`rowsUsed` between 0 and 2147483647)', + '((`rowLimit` is null) or (`rowsUsed` <= `rowLimit`))' + ], + seed: 'bootstrap' + }, + { + name: 'snapshot_journal_retention', + columns: [ + integer('id'), + column('floor', 'varchar(19)'), + integer('receiptLimit'), + column('receiptLifetimeMs', 'bigint') + ], + indexes: [primary('id')], + checks: ['(`id` = 1)', '(`receiptLimit` between 1 and 128)', '(`receiptLifetimeMs` between 1 and 2592000000)'], + seed: 'retention', + charset: 'ascii' + }, + { + name: 'snapshot_journal_receipts', + columns: [ + column('requestId', 'varchar(64)'), + column('binding', 'varchar(64)'), + column('highWater', 'varchar(19)'), + column('floor', 'varchar(19)'), + column('expiresAt', 'bigint') + ], + indexes: [ + primary('requestId'), + { name: 'snapshot_journal_receipts_expiry', columns: ['expiresAt', 'requestId'], unique: false } + ], + checks: ['(`expiresAt` between 1 and 9007199254740991)'], + charset: 'ascii' + } + ] + return specs.map((spec, i) => ({ + ...spec, + sql: spec.charset === 'ascii' ? ddl[i] : ddl[i].replace(/ ENGINE=InnoDB$/, '') + tail + })) +} + +async function plan( + k: Knex, + ceiling: SnapshotJournalRevision, + receiptPolicy: SnapshotJournalReceiptPolicy, + config?: Knex.MigratorConfig +): Promise { + const policy = snapshotJournalReceiptPolicy(receiptPolicy) + client(k) + if (snapshotJournalRevision(ceiling) === '0') return invalid() + const source = await readSnapshotJournalMysqlBinding(k, config) + const [contexts]: Context[][] = await k.raw( + 'SELECT @@sql_mode sqlMode,@@character_set_client charset,@@collation_connection collation,@@collation_database databaseCollation,CURRENT_USER() definer' + ) + const context = contexts[0] + const contextKeys: Array = ['sqlMode', 'charset', 'collation', 'databaseCollation', 'definer'] + if ( + contexts.length !== 1 || + context === null || + typeof context !== 'object' || + Array.isArray(context) || + Object.keys(context).length !== contextKeys.length || + contextKeys.some(key => typeof context[key] !== 'string' || context[key].length > 4096) + ) + return invalid() + const triggers = (await snapshotJournalMysqlObserverSql(k)).map(sql => { + const match = + /^CREATE TRIGGER (snapshot_journal_\w+) AFTER (INSERT|UPDATE|DELETE) ON `([a-z_]+)` FOR EACH ROW (BEGIN .*)$/s.exec( + sql + ) + if (!match) return invalid() + return { name: match[1], event: match[2], table: match[3], body: match[4], sql } + }) + const objects: ObjectDefinition[] = [ + ...tables(k).map(definition => ({ type: 'table' as const, definition })), + ...triggers.map(definition => ({ type: 'trigger' as const, definition })) + ] + if (objects.length !== objectCount || new Set(objects.map(object => object.definition.name)).size !== objects.length) + return invalid() + const digest = createHash('sha256') + .update('snapshot-journal-mysql-plan-v1\n') + .update(JSON.stringify([objects, context, policy])) + .digest('hex') + return { binding: { source, ceiling, plan: digest }, objects, context, receiptPolicy: policy } +} + +async function reserved(k: Knex): Promise> { + const [tables]: Array> = await k.raw( + "SELECT TABLE_NAME name,TABLE_TYPE type FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(LEFT(TABLE_NAME,17))='snapshot_journal_' ORDER BY TABLE_NAME LIMIT 130" + ) + const [triggers]: Array> = await k.raw( + "SELECT TRIGGER_NAME name,'TRIGGER' type FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND LOWER(LEFT(TRIGGER_NAME,17))='snapshot_journal_' ORDER BY TRIGGER_NAME LIMIT 130" + ) + if (tables.length + triggers.length > objectCount + 1) return invalid() + return [...tables, ...triggers] +} +const owner = (epoch: string): string => 'snapshot-journal-owner:' + epoch +function triggerBody(trigger: Trigger, epoch: string): string { + return trigger.body.replace(/^BEGIN /, 'BEGIN /* ' + owner(epoch) + ' */ ') +} + +async function validateTable(k: Knex, table: Table, epoch: string): Promise { + const charset = table.charset ?? 'utf8mb4' + const [actual]: Array< + Array<{ + engine: string + type: string + collation: string + rowFormat: string + options: string + comment: string + }> + > = await k.raw( + 'SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?', + [table.name] + ) + if ( + actual.length !== 1 || + actual[0].engine !== 'InnoDB' || + actual[0].type !== 'BASE TABLE' || + actual[0].collation !== charset + '_bin' || + actual[0].rowFormat !== 'Dynamic' || + actual[0].options !== 'row_format=DYNAMIC' || + actual[0].comment !== owner(epoch) + ) + return invalid() + const [columns]: Array< + Array<{ + name: string + type: string + nullable: string + defaultValue: unknown + extra: string + charset: string | null + collation: string | null + expression: string + }> + > = await k.raw( + 'SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9', + [table.name] + ) + if ( + columns.length !== table.columns.length || + columns.some((actual, i) => { + const expected = table.columns[i], + text = expected.type === 'text' || expected.type.startsWith('varchar') + return ( + actual.name !== expected.name || + actual.type.replace(/^(int|tinyint|bigint)\(\d+\)/, '$1') !== expected.type || + actual.nullable !== (expected.nullable ? 'YES' : 'NO') || + actual.defaultValue !== null || + actual.extra !== (expected.auto ? 'auto_increment' : '') || + actual.charset !== (text ? charset : null) || + actual.collation !== (text ? charset + '_bin' : null) || + actual.expression !== '' + ) + }) + ) + return invalid() + const expectedIndexes = table.indexes.flatMap(index => + index.columns.map((columnName, i) => ({ + name: index.name, + columnName, + position: i + 1, + nonUnique: index.unique ? 0 : 1, + direction: 'A', + prefix: null, + type: 'BTREE', + visible: 'YES', + expression: null + })) + ) + const [indexes]: Array>> = await k.raw( + 'SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16', + [table.name] + ) + if (JSON.stringify(indexes) !== JSON.stringify(expectedIndexes)) return invalid() + const [constraints]: Array> = + await k.raw( + 'SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8', + [table.name] + ) + const expectedConstraints = [ + { name: 'PRIMARY', type: 'PRIMARY KEY', enforced: 'YES', clause: null }, + ...table.checks.map((clause, i) => ({ + name: table.name + '_chk_' + (i + 1), + type: 'CHECK', + enforced: 'YES', + clause: clause.replaceAll("'", String.raw`\'`) + })) + ] + if (JSON.stringify(constraints) !== JSON.stringify(expectedConstraints)) return invalid() + const [triggers]: unknown[][] = await k.raw( + 'SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1', + [table.name] + ) + const [partitions]: unknown[][] = await k.raw( + 'SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1', + [table.name] + ) + const [foreign]: unknown[][] = await k.raw( + 'SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1', + [table.name] + ) + if (triggers.length || partitions.length || foreign.length) return invalid() +} + +async function validateRetention( + k: Knex, + state: SnapshotJournalMysqlIntent, + policy: SnapshotJournalReceiptPolicy +): Promise { + const rows = await k('snapshot_journal_retention') + .select( + 'id', + 'receiptLimit', + k.raw('CAST(receiptLifetimeMs AS CHAR) receiptLifetimeMs'), + k.raw('SUBSTRING(floor,1,20) floor') + ) + .limit(2) + if ( + rows.length !== 1 || + rows[0].id !== 1 || + rows[0].receiptLimit !== policy.receiptLimit || + rows[0].receiptLifetimeMs !== String(policy.receiptLifetimeMs) + ) + return invalid() + const floor = snapshotJournalRevision(rows[0].floor) + if (compareSnapshotJournalRevisions(floor, state.ceiling) > 0 || (state.nextObject < objectCount && floor !== '0')) + return invalid() +} + +async function validateObject( + k: Knex, + object: ObjectDefinition, + state: SnapshotJournalMysqlIntent, + context: Context, + policy: SnapshotJournalReceiptPolicy +): Promise { + if (object.type === 'table') { + await validateTable(k, object.definition, state.epoch) + if (object.definition.seed === 'clock') { + const clocks = await k('snapshot_journal_clock').select('id', k.raw('CAST(ceiling AS CHAR) ceiling')).limit(2) + if (clocks.length !== 1 || clocks[0].id !== 1 || clocks[0].ceiling !== state.ceiling) return invalid() + } + if (object.definition.seed === 'bootstrap' && state.nextObject < objectCount) { + const progress = await k('snapshot_journal_bootstrap').select('*').limit(2) + if ( + progress.length !== 1 || + progress[0].id !== 1 || + progress[0].stream !== 0 || + progress[0].cursor !== null || + progress[0].rowLimit !== null || + progress[0].rowsUsed !== 0 + ) + return invalid() + } + if (object.definition.seed === 'retention') await validateRetention(k, state, policy) + return + } + const definition = object.definition, + expectedBody = triggerBody(definition, state.epoch) + const [rows]: Array< + Array<{ + name: string + table: string + event: string + timing: string + body: string + sqlMode: string + charset: string + collation: string + databaseCollation: string + definer: string + }> + > = await k.raw( + 'SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?', + [expectedBody.length + 1, definition.name] + ) + if ( + rows.length !== 1 || + rows[0].name !== definition.name || + rows[0].table !== definition.table || + rows[0].event !== definition.event || + rows[0].timing !== 'AFTER' || + rows[0].body !== expectedBody || + Object.entries(context).some(([key, value]) => rows[0][key as keyof Context] !== value) + ) + return invalid() +} + +async function validateObjects(k: Knex, p: Plan, state: SnapshotJournalMysqlIntent): Promise { + if (state.nextObject > p.objects.length || (state.complete && state.nextObject !== p.objects.length)) return invalid() + const actual = await reserved(k) + const intent = actual.filter(object => object.name === SNAPSHOT_JOURNAL_MYSQL_INTENT && object.type === 'BASE TABLE') + if (intent.length !== 1) return invalid() + const expected = p.objects.slice(0, state.nextObject + 1) + if ( + actual.some( + object => + object.name !== SNAPSHOT_JOURNAL_MYSQL_INTENT && + !expected.some( + value => + value.definition.name === object.name && object.type === (value.type === 'table' ? 'BASE TABLE' : 'TRIGGER') + ) + ) + ) + return invalid() + let currentExists = false + await runInSeries(p.objects.entries(), async ([i, object]) => { + const found = actual.filter(row => row.name === object.definition.name) + if ((i < state.nextObject && found.length !== 1) || found.length > 1) return invalid() + if (found.length === 1) { + await validateObject(k, object, state, p.context, p.receiptPolicy) + if (i === state.nextObject) currentExists = true + } + }) + return currentExists +} + +async function validateState( + k: Knex, + p: Plan, + state: SnapshotJournalMysqlIntent +): Promise { + if (await k('snapshot_journal_events').first('revision')) return invalid() + const receipts = await k('snapshot_journal_receipts') + .select(k.raw('1 AS occupied')) + .limit(p.receiptPolicy.receiptLimit + 1) + if (receipts.length > p.receiptPolicy.receiptLimit) return invalid() + const clocks = await k('snapshot_journal_clock').select('id', k.raw('CAST(ceiling AS CHAR) ceiling')).limit(2) + if (clocks.length !== 1 || clocks[0].id !== 1 || clocks[0].ceiling !== p.binding.ceiling) return invalid() + const progress = await k('snapshot_journal_bootstrap').select('*').limit(2) + if (progress.length !== 1) return invalid() + const row = progress[0] + if ( + row.id !== 1 || + !Number.isInteger(row.stream) || + row.stream < 0 || + row.stream > 17 || + !(row.cursor === null || (typeof row.cursor === 'string' && Buffer.byteLength(row.cursor, 'utf8') <= 2048)) || + (row.stream === 17 && row.cursor !== null) || + (state.complete && row.stream !== 17) || + !validSnapshotJournalBootstrapBudget(row) || + (row.rowLimit === null && (row.stream !== 0 || row.cursor !== null)) + ) + return invalid() + const invalidations = await k('snapshot_journal_invalid').select('*').limit(2) + if ( + invalidations.length > 1 || + invalidations.some( + row => row.id !== 1 || !['capacity-exhausted', 'revision-exhausted', 'key-out-of-range'].includes(row.reason) + ) + ) + return invalid() + return { ...state, enabled: invalidations.length === 0 } +} + +/** Excludes concurrent migrators and schema changes. Every implicit-DDL step is + * preceded by durable intent; only that step may exist without its acknowledgement. + * Epoch markers and complete typed metadata are required on every resume. No SQL + * is loaded from persisted state, and no existing source/object is dropped. + */ +export async function installSnapshotJournalMysqlGeneration( + k: Knex, + ceiling: SnapshotJournalRevision, + receiptPolicy: SnapshotJournalReceiptPolicy, + config?: Knex.MigratorConfig +): Promise { + if (k.isTransaction) return invalid() + const p = await plan(k, ceiling, receiptPolicy, config) + const existing = await reserved(k) + let state = + existing.length === 0 + ? await createSnapshotJournalMysqlIntent(k, p.binding) + : await readSnapshotJournalMysqlIntent(k, p.binding) + const currentExists = await validateObjects(k, p, state) + const initial = state.nextObject + await runInSeries(p.objects.entries(), async ([i, object]) => { + if (i < initial) return + if (!(i === initial && currentExists)) { + if (object.type === 'trigger') + await k.raw(object.definition.sql.replace(object.definition.body, triggerBody(object.definition, state.epoch))) + else { + const table = object.definition + const sql = table.sql + " COMMENT='" + owner(state.epoch) + "'" + if (table.seed === 'clock') await k.raw(sql + ' SELECT 1 id,? ceiling', [ceiling]) + else if (table.seed === 'bootstrap') + await k.raw(sql + ' SELECT 1 id,0 stream,NULL `cursor`,NULL rowLimit,0 rowsUsed') + else if (table.seed === 'retention') + await k.raw(sql + " SELECT 1 id,'0' floor,? receiptLimit,? receiptLifetimeMs", [ + p.receiptPolicy.receiptLimit, + p.receiptPolicy.receiptLifetimeMs + ]) + else await k.raw(sql) + } + await validateObject(k, object, state, p.context, p.receiptPolicy) + } + const updated = await k(SNAPSHOT_JOURNAL_MYSQL_INTENT) + .where({ id: 1, epoch: state.epoch, nextObject: i, complete: 0 }) + .update({ nextObject: i + 1 }) + if (updated !== 1) return invalid() + state = { ...state, nextObject: i + 1 } + }) + if ((await readSnapshotJournalMysqlBinding(k, config)) !== p.binding.source) return invalid() + await validateObjects(k, p, state) + return await validateState(k, p, state) +} + +export async function readSnapshotJournalMysqlGeneration( + k: Knex, + ceiling: SnapshotJournalRevision, + receiptPolicy: SnapshotJournalReceiptPolicy, + config?: Knex.MigratorConfig +): Promise { + const p = await plan(k, ceiling, receiptPolicy, config), + state = await readSnapshotJournalMysqlIntent(k, p.binding) + if (state.nextObject !== p.objects.length) return invalid() + await validateObjects(k, p, state) + return await validateState(k, p, state) +} + +/** Publication still requires the registered migration journal in the reader view. */ +export async function completeSnapshotJournalMysqlGeneration( + k: Knex, + ceiling: SnapshotJournalRevision, + receiptPolicy: SnapshotJournalReceiptPolicy, + config?: Knex.MigratorConfig +): Promise { + const validated = await readSnapshotJournalMysqlGeneration(k, ceiling, receiptPolicy, config) + return await k.transaction(async t => { + if (!(await t('snapshot_journal_clock').where('id', 1).forUpdate().noWait().first('id'))) return invalid() + const state = await readSnapshotJournalMysqlIntent(t, validated) + if (state.epoch !== validated.epoch || state.nextObject !== objectCount) return invalid() + const progress = await t('snapshot_journal_bootstrap').where('id', 1).first('stream', 'cursor') + if ( + progress?.stream !== 17 || + progress.cursor !== null || + (await t('snapshot_journal_invalid').where('id', 1).first('id')) + ) + return invalid() + const updated = await t(SNAPSHOT_JOURNAL_MYSQL_INTENT) + .where({ id: 1, epoch: state.epoch, nextObject: objectCount }) + .update({ complete: 1 }) + if (updated !== 1) return invalid() + return { ...state, complete: true, enabled: true } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlIntent.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlIntent.test.ts new file mode 100644 index 000000000..ef35c456a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlIntent.test.ts @@ -0,0 +1,258 @@ +import { knex, type Knex } from 'knex' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { + createSnapshotJournalMysqlIntent, + readSnapshotJournalMysqlIntent, + type SnapshotJournalMysqlBinding +} from './SnapshotJournalMysqlIntent' +import { snapshotJournalRevision } from './SnapshotJournalRevision' +let binding: SnapshotJournalMysqlBinding +beforeEach(() => { + binding = { + source: 'a'.repeat(64), + plan: 'b'.repeat(64), + ceiling: snapshotJournalRevision('9223372036854775807') + } +}) +const epoch = '12345678-1234-4123-8123-123456789abc' +const captured: Record>> = JSON.parse( + readFileSync(join(__dirname, '../../../../test/fixtures/snapshotJournal/mysql-intent-metadata-fixture.json'), 'utf8') +) +async function fixture() { + const database = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + await database.raw( + 'CREATE TABLE snapshot_journal_generation(id INTEGER,version INTEGER,epoch TEXT,source TEXT,plan TEXT,ceiling TEXT,nextObject INTEGER,complete INTEGER)' + ) + await database('snapshot_journal_generation').insert({ + id: 1, + version: 1, + epoch, + ...binding, + nextObject: 0, + complete: 0 + }) + const metadata = structuredClone(captured), + configuration = { client: { config: { client: 'mysql2' } }, isTransaction: false }, + version = { value: '8.4.0' } + const raw = jest.fn(async (sql: string, values?: unknown[]) => { + if (sql === 'SELECT VERSION() version') return [[{ version: version.value }]] + for (const [table, key] of [ + ['TABLES', 'tables'], + ['COLUMNS', 'columns'], + ['STATISTICS', 'indexes'], + ['TABLE_CONSTRAINTS', 'constraints'], + ['TRIGGERS', 'triggers'] + ]) + if (sql.includes('information_schema.' + table + ' ')) return [metadata[key]] + // Driver boundary only; native MySQL separately executes the atomic CTAS. + if (sql.startsWith('CREATE TABLE snapshot_journal_generation')) { + if (!values) throw new Error('Missing bound intent values') + await database('snapshot_journal_generation').update({ + epoch: values[0], + source: values[1], + plan: values[2], + ceiling: values[3] + }) + return {} + } + throw new Error('Unexpected SQL fixture request') + }) + const k = Object.assign((table: string) => database(table), configuration, { + raw + }) as unknown as Knex + return { k, database, metadata, raw, version } +} +test('native metadata and exact signed63 binding resume the persisted epoch', async () => { + const f = await fixture() + try { + expect(await readSnapshotJournalMysqlIntent(f.k, binding)).toEqual({ + ...binding, + epoch, + nextObject: 0, + complete: false + }) + await f.database('snapshot_journal_generation').update({ nextObject: 128, complete: 1 }) + expect(await readSnapshotJournalMysqlIntent(f.k, binding)).toEqual({ + ...binding, + epoch, + nextObject: 128, + complete: true + }) + } finally { + await f.database.destroy() + } +}) +test.each(['8.0.21', '8.0.40-commercial', '8.4.0', '8.10.0'])( + 'supported %s binds intent values and reads back its newly generated epoch', + async version => { + const f = await fixture() + try { + f.version.value = version + const created = await createSnapshotJournalMysqlIntent(f.k, binding) + expect(created).toMatchObject({ ...binding, nextObject: 0, complete: false }) + expect(created.epoch).not.toBe(epoch) + expect(created.epoch).toMatch(/^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/) + expect(f.raw.mock.calls.filter(([sql]) => sql.startsWith('CREATE TABLE'))).toHaveLength(1) + } finally { + await f.database.destroy() + } + } +) +test.each(['8.0.20', '5.7.44', '10.11.8-MariaDB', '9.0.0', 'unknown', 'prefix-8.4.0'])( + 'unsupported atomic-DDL baseline %s performs no DDL', + async version => { + const f = await fixture() + try { + f.version.value = version + await expect(createSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + expect(f.raw.mock.calls.map(([sql]) => sql)).toEqual(['SELECT VERSION() version']) + } finally { + await f.database.destroy() + } + } +) +test.each(['sqlite3', 'better-sqlite3', 'pg'])('unsupported driver %s refuses without I/O', async client => { + const f = await fixture() + try { + f.k.client.config.client = client + await expect(createSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + await expect(readSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + expect(f.raw).not.toHaveBeenCalled() + } finally { + await f.database.destroy() + } +}) +test('implicit DDL never commits a caller-owned transaction', async () => { + const f = await fixture() + try { + Object.defineProperty(f.k, 'isTransaction', { value: true }) + await expect(createSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + expect(f.raw).not.toHaveBeenCalled() + } finally { + await f.database.destroy() + } +}) +test.each([ + { source: 'A'.repeat(64) }, + { source: 'x' + 'a'.repeat(64) }, + { source: 'a'.repeat(64) + 'x' }, + { plan: 'x' + 'b'.repeat(64) }, + { plan: 'b'.repeat(64) + 'x' }, + { plan: 'bad' }, + { source: { toString: () => 'a'.repeat(64) } }, + { plan: { toString: () => 'b'.repeat(64) } }, + { ceiling: '0' }, + { ceiling: '01' }, + { ceiling: '9223372036854775808' } +])('invalid binding %j refuses before SQL', async patch => { + const f = await fixture() + try { + await expect( + createSnapshotJournalMysqlIntent(f.k, { ...binding, ...patch } as SnapshotJournalMysqlBinding) + ).rejects.toThrow() + expect(f.raw).not.toHaveBeenCalled() + } finally { + await f.database.destroy() + } +}) +test.each([ + ['tables', 'engine', 'MyISAM'], + ['tables', 'type', 'VIEW'], + ['tables', 'collation', 'ascii_general_ci'], + ['tables', 'rowFormat', 'Compact'], + ['tables', 'options', ''], + ['columns', 'name', 'foreign'], + ['columns', 'type', 'bigint'], + ['columns', 'nullable', 'YES'], + ['columns', 'defaultValue', 0], + ['columns', 'extra', 'auto_increment'], + ['columns', 'expression', '1'], + ['columns', 'charset', 'utf8mb4'], + ['columns', 'collation', 'utf8mb4_bin'], + ['indexes', 'name', 'foreign'], + ['indexes', 'columnName', 'source'], + ['indexes', 'nonUnique', 1], + ['indexes', 'direction', 'D'], + ['indexes', 'prefix', 1], + ['indexes', 'type', 'HASH'], + ['indexes', 'visible', 'NO'], + ['constraints', 'name', 'foreign'], + ['constraints', 'type', 'CHECK'], + ['constraints', 'enforced', 'NO'] +])('native %s %s drift refuses ownership', async (key, field, value) => { + const f = await fixture() + try { + f.metadata[key as string][0][field as string] = value + await expect(readSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + } finally { + await f.database.destroy() + } +}) +test.each(['tables', 'columns', 'indexes', 'constraints'])('incomplete %s metadata refuses', async key => { + const f = await fixture() + try { + f.metadata[key].pop() + await expect(readSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + } finally { + await f.database.destroy() + } +}) +test('an attached foreign observer refuses ownership', async () => { + const f = await fixture() + try { + f.metadata.triggers.push({ TRIGGER_NAME: 'foreign' }) + await expect(readSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + } finally { + await f.database.destroy() + } +}) +test.each([ + { id: 2 }, + { version: 2 }, + { epoch: 'wrong' }, + { epoch: 'x' + epoch }, + { epoch: epoch + 'x' }, + { source: 'c'.repeat(64) }, + { plan: 'c'.repeat(64) }, + { ceiling: '1' }, + { nextObject: -1 }, + { nextObject: 129 }, + { nextObject: 0.5 }, + { complete: 2 } +])('persisted state %j cannot establish installation continuity', async patch => { + const f = await fixture() + try { + await f.database('snapshot_journal_generation').update(patch) + await expect(readSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + } finally { + await f.database.destroy() + } +}) +test.each(['missing', 'extra'])('%s intent row refuses', async kind => { + const f = await fixture() + try { + if (kind === 'missing') await f.database('snapshot_journal_generation').delete() + else + await f + .database('snapshot_journal_generation') + .insert({ ...(await f.database('snapshot_journal_generation').first()), id: 2 }) + await expect(readSnapshotJournalMysqlIntent(f.k, binding)).rejects.toThrow('Invalid or unowned') + } finally { + await f.database.destroy() + } +}) + +test('legacy MySQL alias can resume its exact persisted intent', async () => { + const f = await fixture() + try { + f.k.client.config.client = 'mysql' + expect(await readSnapshotJournalMysqlIntent(f.k, binding)).toMatchObject({ ...binding, epoch }) + } finally { + await f.database.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlIntent.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlIntent.ts new file mode 100644 index 000000000..c2dcd686a --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlIntent.ts @@ -0,0 +1,185 @@ +import type { Knex } from 'knex' +import { randomUUID } from 'node:crypto' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' + +export const SNAPSHOT_JOURNAL_MYSQL_INTENT = 'snapshot_journal_generation' +export interface SnapshotJournalMysqlBinding { + source: string + plan: string + ceiling: SnapshotJournalRevision +} +export interface SnapshotJournalMysqlIntent extends SnapshotJournalMysqlBinding { + epoch: string + nextObject: number + complete: boolean +} +const columns = [ + ['id', 'int'], + ['version', 'int'], + ['epoch', 'varchar(36)'], + ['source', 'varchar(64)'], + ['plan', 'varchar(64)'], + ['ceiling', 'varchar(19)'], + ['nextObject', 'int'], + ['complete', 'tinyint'] +] as const +export const SNAPSHOT_JOURNAL_MYSQL_INTENT_DDL = + 'CREATE TABLE snapshot_journal_generation(id INTEGER NOT NULL PRIMARY KEY,version INTEGER NOT NULL,epoch VARCHAR(36) NOT NULL,source VARCHAR(64) NOT NULL,plan VARCHAR(64) NOT NULL,ceiling VARCHAR(19) NOT NULL,nextObject INTEGER NOT NULL,complete BOOLEAN NOT NULL) ENGINE=InnoDB DEFAULT CHARACTER SET ascii COLLATE ascii_bin ROW_FORMAT=DYNAMIC' +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid or unowned MySQL snapshot journal installation intent') +} +function binding(value: SnapshotJournalMysqlBinding): void { + if ( + typeof value.source !== 'string' || + typeof value.plan !== 'string' || + !/^[0-9a-f]{64}$/.test(value.source) || + !/^[0-9a-f]{64}$/.test(value.plan) || + snapshotJournalRevision(value.ceiling) === '0' + ) + invalid() +} +function client(k: Knex): void { + if (k.client.config.client !== 'mysql' && k.client.config.client !== 'mysql2') invalid() +} +async function structure(k: Knex): Promise { + const [tables]: Array< + Array<{ engine: string; type: string; collation: string; rowFormat: string; options: string }> + > = await k.raw( + 'SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?', + [SNAPSHOT_JOURNAL_MYSQL_INTENT] + ) + if ( + tables.length !== 1 || + tables[0].engine !== 'InnoDB' || + tables[0].type !== 'BASE TABLE' || + tables[0].collation !== 'ascii_bin' || + tables[0].rowFormat !== 'Dynamic' || + tables[0].options !== 'row_format=DYNAMIC' + ) + return invalid() + const [actual]: Array< + Array<{ + name: string + type: string + nullable: string + defaultValue: unknown + extra: string + charset: string | null + collation: string | null + expression: string + }> + > = await k.raw( + 'SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9', + [SNAPSHOT_JOURNAL_MYSQL_INTENT] + ) + if ( + actual.length !== columns.length || + actual.some((column, index) => { + const [name, type] = columns[index], + text = type.startsWith('varchar') + return ( + column.name !== name || + column.type.replace(/^(int|tinyint)\(\d+\)$/, '$1') !== type || + column.nullable !== 'NO' || + column.defaultValue !== null || + column.extra !== '' || + column.expression !== '' || + column.charset !== (text ? 'ascii' : null) || + column.collation !== (text ? 'ascii_bin' : null) + ) + }) + ) + return invalid() + const [indexes]: Array< + Array<{ + name: string + columnName: string + nonUnique: number + direction: string + prefix: unknown + type: string + visible: string + }> + > = await k.raw( + 'SELECT INDEX_NAME name,COLUMN_NAME columnName,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 2', + [SNAPSHOT_JOURNAL_MYSQL_INTENT] + ) + if ( + indexes.length !== 1 || + indexes[0].name !== 'PRIMARY' || + indexes[0].columnName !== 'id' || + indexes[0].nonUnique !== 0 || + indexes[0].direction !== 'A' || + indexes[0].prefix !== null || + indexes[0].type !== 'BTREE' || + indexes[0].visible !== 'YES' + ) + return invalid() + const [constraints]: Array> = await k.raw( + 'SELECT CONSTRAINT_NAME name,CONSTRAINT_TYPE type,ENFORCED enforced FROM information_schema.TABLE_CONSTRAINTS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? LIMIT 2', + [SNAPSHOT_JOURNAL_MYSQL_INTENT] + ) + if ( + constraints.length !== 1 || + constraints[0].name !== 'PRIMARY' || + constraints[0].type !== 'PRIMARY KEY' || + constraints[0].enforced !== 'YES' + ) + return invalid() + const [triggers]: Array = await k.raw( + 'SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1', + [SNAPSHOT_JOURNAL_MYSQL_INTENT] + ) + if (triggers.length !== 0) return invalid() +} +export async function readSnapshotJournalMysqlIntent( + k: Knex, + expected: SnapshotJournalMysqlBinding +): Promise { + client(k) + binding(expected) + await structure(k) + const rows = await k(SNAPSHOT_JOURNAL_MYSQL_INTENT).select('*').limit(2) + if (rows.length !== 1) return invalid() + const row = rows[0] + if ( + row.id !== 1 || + row.version !== 1 || + typeof row.epoch !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(row.epoch) || + row.source !== expected.source || + row.plan !== expected.plan || + row.ceiling !== expected.ceiling || + !Number.isInteger(row.nextObject) || + row.nextObject < 0 || + row.nextObject > 128 || + ![0, 1].includes(row.complete) + ) + return invalid() + return { ...expected, epoch: row.epoch, nextObject: row.nextObject, complete: row.complete === 1 } +} + +/** MySQL 8.0.21+ InnoDB atomic CREATE TABLE SELECT persists the first intent row + * with its table. Caller excludes other migrators and validates source/schema + * before calling; no existing object is adopted, replaced or dropped here. + * This is not a transaction wrapper: ordinary MySQL DDL commits implicitly. + */ +export async function createSnapshotJournalMysqlIntent( + k: Knex, + expected: SnapshotJournalMysqlBinding +): Promise { + client(k) + binding(expected) + if (k.isTransaction) return invalid() + const [[server]]: Array> = await k.raw('SELECT VERSION() version') + const version = /^8\.(\d+)\.(\d+)(?:[-.]|$)/.exec(server.version) + if (!version || (Number(version[1]) === 0 && Number(version[2]) < 21)) return invalid() + const epoch = randomUUID() + await k.raw( + SNAPSHOT_JOURNAL_MYSQL_INTENT_DDL + + ' SELECT 1 id,1 version,? epoch,? source,? plan,? ceiling,0 nextObject,0 complete', + [epoch, expected.source, expected.plan, expected.ceiling] + ) + return await readSnapshotJournalMysqlIntent(k, expected) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlObservers.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlObservers.ts new file mode 100644 index 000000000..48c94f130 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlObservers.ts @@ -0,0 +1,159 @@ +import type { Knex } from 'knex' +import { runInSeries } from '../../../utility/runInSeries' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { numeric } from '../../schema/snapshotSqliteMembership' +import { + SNAPSHOT_JOURNAL_MYSQL_CLOCK_VARIABLES as variables, + SNAPSHOT_JOURNAL_MYSQL_ADVANCE as advance +} from './SnapshotJournalMysqlClock' + +const tables = [...numeric.map(source => source.table), 'tx_labels_map', 'output_tags_map', 'certificate_fields'] +const q = (name: string) => '`' + name.replaceAll('`', '``') + '`' +const physicalKey = 'tableId,id1,id2,exactText', + scopeKey = 'tableId,userId,id1,id2,exactText' +export const SNAPSHOT_JOURNAL_MYSQL_METADATA_DDL = [ + `CREATE TABLE snapshot_journal_physical(tableId INT NOT NULL,id1 BIGINT UNSIGNED NOT NULL,id2 BIGINT UNSIGNED NOT NULL,exactText VARBINARY(400) NOT NULL,revision BIGINT UNSIGNED NOT NULL,generation BIGINT UNSIGNED NOT NULL,present BOOLEAN NOT NULL,PRIMARY KEY(${physicalKey}),KEY snapshot_journal_physical_page(tableId,revision,id1,id2,exactText)) ENGINE=InnoDB`, + `CREATE TABLE snapshot_journal_scope(tableId INT NOT NULL,userId BIGINT UNSIGNED NOT NULL,id1 BIGINT UNSIGNED NOT NULL,id2 BIGINT UNSIGNED NOT NULL,exactText VARBINARY(400) NOT NULL,revision BIGINT UNSIGNED NOT NULL,present BOOLEAN NOT NULL,PRIMARY KEY(${scopeKey}),KEY snapshot_journal_scope_page(userId,tableId,revision,id1,id2,exactText)) ENGINE=InnoDB` +] +const tuple = (table: string, p: string): string[] => { + const key = numeric.find(source => source.table === table)?.key + if (key) return [p + '.' + q(key), '0', "CAST('' AS BINARY)"] + if (table === 'tx_labels_map') return [p + '.txLabelId', p + '.transactionId', "CAST('' AS BINARY)"] + if (table === 'output_tags_map') return [p + '.outputTagId', p + '.outputId', "CAST('' AS BINARY)"] + return [p + '.certificateId', '0', 'CAST(' + p + '.fieldName AS BINARY)'] +} +const scopeUpsert = (selection: string) => + `INSERT INTO snapshot_journal_scope(${scopeKey},revision,present) ${selection} ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); ` +function keyGuard(key: string[], owner?: string): string { + const [id1, id2, text] = key + const valid = [ + `(${id1} BETWEEN 1 AND 9007199254740991)`, + `(${id2} BETWEEN 0 AND 9007199254740991)`, + `(OCTET_LENGTH(${text})<=400)`, + ...(owner ? [`(${owner} BETWEEN 1 AND 9007199254740991)`] : []) + ].join(' AND ') + return `IF NOT COALESCE((${valid}),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; ` +} + +function sourceScope(tableId: number, keys: string[]): string { + if (tableId < 8) + return scopeUpsert( + `SELECT ${tableId},NEW.userId,NEW.${q(numeric[tableId].key)},0,CAST('' AS BINARY),journalRevision,1` + ) + else if (tableId === 10 || tableId === 11) + return scopeUpsert( + `SELECT ${tableId},snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=${tableId - 10} AND snapshotLeftId=${keys[0]} AND snapshotRightId=${keys[1]} AND snapshotMembership<>0 FOR SHARE` + ) + else if (tableId === 12) + return scopeUpsert( + `SELECT 12,snapshotUserId,snapshotCertificateId,0,CAST(snapshotFieldName AS BINARY),journalRevision,1 FROM snapshot_certificate_field_keys WHERE snapshotCertificateId=NEW.certificateId AND snapshotFieldName=NEW.fieldName AND snapshotMembership<>0 FOR SHARE` + ) + return '' +} +function freshGeneration(event: string, same: string): string { + if (event === 'INSERT') return 'TRUE' + if (event === 'UPDATE') return `NOT (${same})` + return 'FALSE' +} +function physicalTrigger(tableId: number, table: string, event: string, changed: string): string { + const same = tuple(table, 'OLD') + .map((x, i) => `(${x} <=> ${tuple(table, 'NEW')[i]})`) + .join(' AND ') + const write = (p: string, present: number, fresh: string) => + `INSERT INTO snapshot_journal_physical(${physicalKey},revision,generation,present) VALUES(${tableId},${tuple(table, p).join(',')},journalRevision,journalRevision,${present}) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN ${fresh} THEN VALUES(generation) ELSE generation END,present=VALUES(present); ` + const p = event === 'DELETE' ? 'OLD' : 'NEW', + keys = tuple(table, p) + let body = advance + keyGuard(keys, tableId < 8 ? p + '.userId' : undefined) + if (event === 'UPDATE') body += keyGuard(tuple(table, 'OLD'), tableId < 8 ? 'OLD.userId' : undefined) + body += 'IF journalEnabled THEN ' + if (event === 'UPDATE') body += `IF NOT (${same}) THEN ${write('OLD', 0, 'FALSE')} END IF; ` + body += write(p, event === 'DELETE' ? 0 : 1, freshGeneration(event, same)) + if (event !== 'DELETE') body += sourceScope(tableId, keys) + body += 'END IF; ' + return `CREATE TRIGGER snapshot_journal_physical_${tableId}_${event} AFTER ${event} ON ${q(table)} FOR EACH ROW BEGIN ${variables}IF ${event === 'UPDATE' ? changed : 'TRUE'} THEN ${body} END IF; END` +} + +/** Caller validates the completed MySQL membership/source schema before installing these observers. */ +export async function snapshotJournalMysqlObserverSql(k: Knex): Promise { + if (!['mysql', 'mysql2'].includes(k.client.config.client)) + throw new WERR_INVALID_OPERATION('Snapshot journal observers require MySQL') + const definitions: string[] = [] + const membership: Array<{ + table: string + expressions: (p: string) => string[] + present: (p: string) => string + }> = [ + { + table: 'snapshot_profile_keys', + expressions: p => [ + p + '.snapshotTableId', + p + '.snapshotUserId', + p + '.snapshotRowId', + '0', + "CAST('' AS BINARY)" + ], + present: () => '1' + }, + { + table: 'snapshot_relation_keys', + expressions: p => [ + p + '.snapshotTableId+10', + p + '.snapshotUserId', + p + '.snapshotLeftId', + p + '.snapshotRightId', + "CAST('' AS BINARY)" + ], + present: p => '(' + p + '.snapshotMembership<>0)' + }, + { + table: 'snapshot_certificate_field_keys', + expressions: p => [ + '12', + p + '.snapshotUserId', + p + '.snapshotCertificateId', + '0', + 'CAST(' + p + '.snapshotFieldName AS BINARY)' + ], + present: p => '(' + p + '.snapshotMembership<>0)' + }, + { + table: 'snapshot_global_keys', + expressions: p => [ + 'CASE ' + p + '.tableId WHEN 0 THEN 9 ELSE 8 END', + p + '.userId', + p + '.rowId', + '0', + "CAST('' AS BINARY)" + ], + present: p => p + '.present' + } + ] + for (const [i, source] of membership.entries()) + for (const event of ['INSERT', 'DELETE', 'UPDATE']) { + const p = event === 'DELETE' ? 'OLD' : 'NEW', + fields = source.expressions(p) + const condition = + event === 'UPDATE' + ? [ + ...source.expressions('OLD').map((x, j) => `NOT (${x} <=> ${source.expressions('NEW')[j]})`), + `NOT (${source.present('OLD')} <=> ${source.present('NEW')})` + ].join(' OR ') + : 'TRUE' + definitions.push( + `CREATE TRIGGER snapshot_journal_scope_${i}_${event} AFTER ${event} ON ${q(source.table)} FOR EACH ROW BEGIN ${variables}IF ${condition} THEN ${advance}${keyGuard(fields.slice(2), fields[1])}IF journalEnabled THEN ${scopeUpsert('SELECT ' + [...fields, 'journalRevision', event === 'DELETE' ? '0' : source.present(p)].join(','))} END IF; END IF; END` + ) + } + await runInSeries(tables.entries(), async ([tableId, table]) => { + const [columns]: Array> = await k.raw( + 'SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION', + [table] + ) + if (columns.length === 0) throw new WERR_INVALID_OPERATION('Missing snapshot journal source') + const changed = columns + .map(({ name }) => `NOT (CAST(OLD.${q(name)} AS BINARY) <=> CAST(NEW.${q(name)} AS BINARY))`) + .join(' OR ') + for (const event of ['INSERT', 'UPDATE', 'DELETE']) + definitions.push(physicalTrigger(tableId, table, event, changed)) + }) + return definitions +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlSource.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlSource.test.ts new file mode 100644 index 000000000..692a24bee --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlSource.test.ts @@ -0,0 +1,523 @@ +import { knex, type Knex } from 'knex' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import * as profile from '../../schema/snapshotProfileIndexMigration' +import * as relation from '../../schema/snapshotRelationIndexMigration' +import * as certificate from '../../schema/snapshotCertificateIndexMigration' +import * as global from '../../schema/snapshotGlobalIndexMigration' +import { validateSnapshotJournalMysqlSource, readSnapshotJournalMysqlBinding } from './SnapshotJournalMysqlSource' + +// Native MySQL 8.4 metadata from a real registered migration, queried through a +// SQLite information_schema fixture so predicates/order/limits execute in tests. +// Native qualification separately verifies MySQL's actual metadata semantics. +const fixture: Record>> = JSON.parse( + readFileSync(join(__dirname, '../../../../test/fixtures/snapshotJournal/mysql-source-metadata-fixture.json'), 'utf8') +) +const families = [ + [ + 'TABLES', + 'tables', + { + name: 'TABLE_NAME', + engine: 'ENGINE', + type: 'TABLE_TYPE', + collation: 'TABLE_COLLATION', + rowFormat: 'ROW_FORMAT', + options: 'CREATE_OPTIONS' + } + ], + [ + 'COLUMNS', + 'columns', + { + tableName: 'TABLE_NAME', + name: 'COLUMN_NAME', + position: 'ORDINAL_POSITION', + type: 'COLUMN_TYPE', + nullable: 'IS_NULLABLE', + defaultValue: 'COLUMN_DEFAULT', + extra: 'EXTRA', + charset: 'CHARACTER_SET_NAME', + collation: 'COLLATION_NAME', + expression: 'GENERATION_EXPRESSION' + } + ], + [ + 'STATISTICS', + 'indexes', + { + tableName: 'TABLE_NAME', + name: 'INDEX_NAME', + position: 'SEQ_IN_INDEX', + columnName: 'COLUMN_NAME', + nonUnique: 'NON_UNIQUE', + direction: 'COLLATION', + prefix: 'SUB_PART', + nullable: 'NULLABLE', + type: 'INDEX_TYPE', + visible: 'IS_VISIBLE', + expression: 'EXPRESSION' + } + ], + [ + 'KEY_COLUMN_USAGE', + 'foreignKeys', + { + tableName: 'TABLE_NAME', + name: 'CONSTRAINT_NAME', + position: 'ORDINAL_POSITION', + columnName: 'COLUMN_NAME', + foreignSchema: 'REFERENCED_TABLE_SCHEMA', + foreignTable: 'REFERENCED_TABLE_NAME', + foreignColumn: 'REFERENCED_COLUMN_NAME' + } + ], + [ + 'REFERENTIAL_CONSTRAINTS', + 'foreignRules', + { + tableName: 'TABLE_NAME', + name: 'CONSTRAINT_NAME', + matchOption: 'MATCH_OPTION', + updateRule: 'UPDATE_RULE', + deleteRule: 'DELETE_RULE' + } + ], + [ + 'TRIGGERS', + 'triggers', + { + tableName: 'EVENT_OBJECT_TABLE', + name: 'TRIGGER_NAME', + event: 'EVENT_MANIPULATION', + timing: 'ACTION_TIMING', + actionOrder: 'ACTION_ORDER', + definer: 'DEFINER', + body: 'ACTION_STATEMENT', + sqlMode: 'SQL_MODE', + charset: 'CHARACTER_SET_CLIENT', + connectionCollation: 'COLLATION_CONNECTION', + databaseCollation: 'DATABASE_COLLATION' + } + ] +] as const +let reads: Array +beforeEach(() => { + reads = [ + jest.spyOn(profile, 'readSnapshotProfileIndexState'), + jest.spyOn(relation, 'readSnapshotRelationIndexState'), + jest.spyOn(certificate, 'readSnapshotCertificateIndexState'), + jest.spyOn(global, 'readSnapshotGlobalIndexState') + ] + for (const read of reads) read.mockResolvedValue(true) +}) +afterEach(() => jest.restoreAllMocks()) +async function metadata(): Promise { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const connection = await k.client.acquireConnection() + try { + connection.function('DATABASE', () => 'ts569_snapshot') + } finally { + await k.client.releaseConnection(connection) + } + await k.raw("ATTACH DATABASE ':memory:' AS information_schema") + for (const [table, key, columns] of families) { + const schemaColumns = ['TABLE_SCHEMA', 'CONSTRAINT_SCHEMA', 'TRIGGER_SCHEMA', ...Object.values(columns)] + await k.raw('CREATE TABLE information_schema.?? (' + schemaColumns.map(() => '??').join(',') + ')', [ + table, + ...schemaColumns + ]) + for (const row of fixture[key]) { + const record: Record = { + TABLE_SCHEMA: 'ts569_snapshot', + CONSTRAINT_SCHEMA: 'ts569_snapshot', + TRIGGER_SCHEMA: 'ts569_snapshot' + } + for (const [field, column] of Object.entries(columns)) record[column] = row[field] + await k(table).withSchema('information_schema').insert(record) + } + } + await k.raw( + 'CREATE TABLE information_schema.TABLE_CONSTRAINTS(TABLE_SCHEMA,CONSTRAINT_SCHEMA,TABLE_NAME,CONSTRAINT_NAME,CONSTRAINT_TYPE,ENFORCED)' + ) + await k.raw('CREATE TABLE information_schema.CHECK_CONSTRAINTS(CONSTRAINT_SCHEMA,CONSTRAINT_NAME,CHECK_CLAUSE)') + const original = k.client.processResponse.bind(k.client) + // A MySQL driver's raw result wraps rows; execute identical SELECT predicates. + jest.spyOn(k.client, 'processResponse').mockImplementation((...args: unknown[]) => { + const response = args[0] as { method: string; sql: string } + const result = original(...args) + return response.method === 'raw' && response.sql.startsWith('SELECT') ? [result] : result + }) + k.client.config.client = 'mysql2' + return k +} + +test('published native metadata produces a stable source binding through real predicates', async () => { + const k = await metadata() + try { + await validateSnapshotJournalMysqlSource(k) + const binding = await readSnapshotJournalMysqlBinding(k) + expect(binding).toMatch(/^[0-9a-f]{64}$/) + expect(await readSnapshotJournalMysqlBinding(k)).toBe(binding) + } finally { + await k.destroy() + } +}) +test.each(['sqlite3', 'better-sqlite3', 'pg', 'missing'])( + 'unsupported source %s refuses before database access', + async client => { + const k = await metadata() + try { + k.client.config.client = client + const queries = jest.fn() + k.on('query', queries) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow('Unsupported or incomplete') + expect(queries).not.toHaveBeenCalled() + for (const read of reads) expect(read).not.toHaveBeenCalled() + } finally { + await k.destroy() + } + } +) +test.each([0, 1, 2, 3])('unpublished prerequisite %s prevents journal admission', async index => { + const k = await metadata() + try { + reads[index].mockResolvedValue(false) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow('Unsupported or incomplete') + } finally { + await k.destroy() + } +}) +test.each(['missing', 'engine', 'view'])('source %s is refused', async kind => { + const k = await metadata() + try { + const table = k('TABLES').withSchema('information_schema').where('TABLE_NAME', 'outputs') + if (kind === 'missing') await table.delete() + else await table.update(kind === 'engine' ? { ENGINE: 'MyISAM' } : { TABLE_TYPE: 'VIEW' }) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow('Unsupported or incomplete') + } finally { + await k.destroy() + } +}) +test.each([ + ['UPDATE_RULE', 'CASCADE'], + ['DELETE_RULE', 'CASCADE'], + ['UPDATE_RULE', 'SET NULL'], + ['DELETE_RULE', 'SET NULL'] +])('%s %s cannot bypass source observers', async (column, value) => { + const k = await metadata() + try { + await k('REFERENTIAL_CONSTRAINTS') + .withSchema('information_schema') + .where('TABLE_NAME', 'outputs') + .update({ [column]: value }) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow('Unsupported or incomplete') + } finally { + await k.destroy() + } +}) +test.each(['missing', 'table', 'event', 'timing', 'body'])('changed prerequisite trigger %s is refused', async kind => { + const k = await metadata() + try { + const query = k('TRIGGERS').withSchema('information_schema').where('TRIGGER_NAME', 'snapshot_profile_0_insert') + if (kind === 'missing') await query.delete() + else + await query.update({ + [{ + table: 'EVENT_OBJECT_TABLE', + event: 'EVENT_MANIPULATION', + timing: 'ACTION_TIMING', + body: 'ACTION_STATEMENT' + }[kind]!]: 'changed' + }) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow('Unsupported or incomplete') + } finally { + await k.destroy() + } +}) +test.each([ + ['COLUMNS', 'COLUMN_DEFAULT', 'literal with spaces'], + ['STATISTICS', 'IS_VISIBLE', 'NO'], + ['REFERENTIAL_CONSTRAINTS', 'MATCH_OPTION', 'changed'], + ['KEY_COLUMN_USAGE', 'REFERENCED_COLUMN_NAME', 'changed'], + ['TABLES', 'ROW_FORMAT', 'COMPACT'] +])('binding retains %s %s semantics', async (table, column, value) => { + const k = await metadata() + try { + const before = await readSnapshotJournalMysqlBinding(k) + const update = k(table).withSchema('information_schema') + if (table === 'STATISTICS') update.where('NON_UNIQUE', 1) + await update.update({ [column]: value }) + expect(await readSnapshotJournalMysqlBinding(k)).not.toBe(before) + } finally { + await k.destroy() + } +}) +test('literal bytes and foreign objects with a journal prefix remain in the binding', async () => { + const k = await metadata() + try { + const before = await readSnapshotJournalMysqlBinding(k) + const original = fixture.triggers[0] + const add: Record = { TRIGGER_SCHEMA: 'ts569_snapshot' } + for (const [field, column] of Object.entries(families[5][2])) add[column] = original[field] + Object.assign(add, { + TRIGGER_NAME: 'snapshot_journal_foreign', + ACTION_STATEMENT: "BEGIN DO 'a b'; END" + }) + await k('TRIGGERS').withSchema('information_schema').insert(add) + const first = await readSnapshotJournalMysqlBinding(k) + expect(first).not.toBe(before) + await k('TRIGGERS') + .withSchema('information_schema') + .where('TRIGGER_NAME', 'snapshot_journal_foreign') + .update({ ACTION_STATEMENT: "BEGIN DO 'a b'; END" }) + expect(await readSnapshotJournalMysqlBinding(k)).not.toBe(first) + } finally { + await k.destroy() + } +}) +test.each(['rows', 'value', 'aggregate'])( + 'oversized %s metadata refuses instead of binding a truncated definition', + async kind => { + const k = await metadata() + try { + if (kind === 'value') + await k('COLUMNS') + .withSchema('information_schema') + .update({ COLUMN_DEFAULT: 'x'.repeat(16385) }) + else if (kind === 'aggregate') + await k('COLUMNS') + .withSchema('information_schema') + .update({ COLUMN_DEFAULT: 'x'.repeat(16384) }) + else { + const original = fixture.indexes[0] + for (let i = 0; i < 513; i++) { + const row: Record = { TABLE_SCHEMA: 'ts569_snapshot' } + for (const [field, column] of Object.entries(families[2][2])) row[column] = original[field] + row.INDEX_NAME = 'bounded' + i + await k('STATISTICS').withSchema('information_schema').insert(row) + } + } + await expect(readSnapshotJournalMysqlBinding(k)).rejects.toThrow('Unsupported or incomplete') + } finally { + await k.destroy() + } + } +) + +test.each([ + 'missing primary', + 'composite primary', + 'prefix identity', + 'descending identity', + 'invisible identity', + 'nullable owner', + 'signed identity', + 'generated owner' +])('unsupported source identity %s refuses before journal installation', async kind => { + const k = await metadata() + try { + const index = k('STATISTICS') + .withSchema('information_schema') + .where({ TABLE_NAME: 'tx_labels', INDEX_NAME: 'PRIMARY' }) + if (kind === 'missing primary') await index.delete() + if (kind === 'composite primary') { + const row = await index.first() + await k('STATISTICS') + .withSchema('information_schema') + .insert({ ...row, SEQ_IN_INDEX: 2, COLUMN_NAME: 'userId' }) + } + if (kind === 'prefix identity') await index.update({ SUB_PART: 1 }) + if (kind === 'descending identity') await index.update({ COLLATION: 'D' }) + if (kind === 'invisible identity') await index.update({ IS_VISIBLE: 'NO' }) + if (kind === 'nullable owner') + await k('COLUMNS') + .withSchema('information_schema') + .where({ TABLE_NAME: 'tx_labels', COLUMN_NAME: 'userId' }) + .update({ IS_NULLABLE: 'YES' }) + if (kind === 'signed identity') + await k('COLUMNS') + .withSchema('information_schema') + .where({ TABLE_NAME: 'tx_labels', COLUMN_NAME: 'txLabelId' }) + .update({ COLUMN_TYPE: 'int' }) + if (kind === 'generated owner') + await k('COLUMNS') + .withSchema('information_schema') + .where({ TABLE_NAME: 'tx_labels', COLUMN_NAME: 'userId' }) + .update({ EXTRA: 'VIRTUAL GENERATED' }) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow('Unsupported or incomplete') + } finally { + await k.destroy() + } +}) + +test.each(['missing', 'non-InnoDB', 'view'])( + 'binding detects source table %s after prerequisite validation', + async kind => { + const k = await metadata(), + process = jest.mocked(k.client.processResponse).getMockImplementation()! + let altered = false + jest.mocked(k.client.processResponse).mockImplementation((...args: unknown[]) => { + const query = args[0] as { sql: string }, + result = process(...args) + if (query.sql.includes('ROW_FORMAT rowFormat')) { + altered = true + if (kind === 'missing') result[0].pop() + if (kind === 'non-InnoDB') result[0][0].engine = 'MyISAM' + if (kind === 'view') result[0][0].type = 'VIEW' + } + return result + }) + try { + await expect(readSnapshotJournalMysqlBinding(k)).rejects.toThrow('Unsupported or incomplete') + expect(altered).toBe(true) + } finally { + await k.destroy() + } + } +) + +test.each(['mysql', 'mysql2'])('MySQL source alias %s accepts explicit display widths', async client => { + const k = await metadata() + k.client.config.client = client + try { + await k('COLUMNS') + .withSchema('information_schema') + .where('COLUMN_TYPE', 'int unsigned') + .update({ COLUMN_TYPE: 'int(10) unsigned' }) + await expect(validateSnapshotJournalMysqlSource(k)).resolves.toBeUndefined() + } finally { + await k.destroy() + } +}) + +test.each(['COLUMNS', 'STATISTICS'])('source validation alone bounds %s metadata at 512 entries', async table => { + const k = await metadata() + try { + const query = () => k(table).withSchema('information_schema') + const sourceTables = [ + 'transactions', + 'outputs', + 'certificates', + 'tx_labels', + 'output_baskets', + 'output_tags', + 'commissions', + 'sync_states', + 'proven_txs', + 'proven_tx_reqs', + 'tx_labels_map', + 'output_tags_map', + 'certificate_fields' + ] + const count = Number((await query().whereIn('TABLE_NAME', sourceTables).count('* AS n').first())!.n) + const example = await query().where('TABLE_NAME', 'outputs').first() + for (let i = count; i < 512; i++) + await query().insert( + table === 'COLUMNS' + ? { ...example, COLUMN_NAME: 'extra_' + i } + : { ...example, INDEX_NAME: 'extra_' + i, NON_UNIQUE: 1 } + ) + await expect(validateSnapshotJournalMysqlSource(k)).resolves.toBeUndefined() + await query().insert( + table === 'COLUMNS' + ? { ...example, COLUMN_NAME: 'overflow' } + : { ...example, INDEX_NAME: 'overflow', NON_UNIQUE: 1 } + ) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow( + 'Unsupported or incomplete MySQL snapshot journal source' + ) + } finally { + await k.destroy() + } +}) + +test('a source owner cannot acquire auto-increment identity semantics', async () => { + const k = await metadata() + try { + await k('COLUMNS') + .withSchema('information_schema') + .where({ TABLE_NAME: 'tx_labels', COLUMN_NAME: 'userId' }) + .update({ EXTRA: 'auto_increment' }) + await expect(validateSnapshotJournalMysqlSource(k)).rejects.toThrow( + 'Unsupported or incomplete MySQL snapshot journal source' + ) + } finally { + await k.destroy() + } +}) + +test('every expected journal observer is excluded from the stable pre-install source binding', async () => { + const k = await metadata() + try { + const before = await readSnapshotJournalMysqlBinding(k) + const original = await k('TRIGGERS').withSchema('information_schema').first() + const names = [ + ...Array.from({ length: 4 }, (_, i) => 'snapshot_journal_scope_' + i), + ...Array.from({ length: 13 }, (_, i) => 'snapshot_journal_physical_' + i) + ].flatMap(prefix => ['INSERT', 'UPDATE', 'DELETE'].map(event => prefix + '_' + event)) + for (const name of names) + await k('TRIGGERS') + .withSchema('information_schema') + .insert({ ...original, TRIGGER_NAME: name, ACTION_STATEMENT: 'BEGIN DO 1; END' }) + expect(await readSnapshotJournalMysqlBinding(k)).toBe(before) + } finally { + await k.destroy() + } +}) + +test.each(['single oversized value', 'multibyte value', '513 rows', 'nonarray'])( + 'binding independently refuses %s from the driver', + async kind => { + const k = await metadata(), + process = jest.mocked(k.client.processResponse).getMockImplementation()! + let altered = false + jest.mocked(k.client.processResponse).mockImplementation((...args: unknown[]) => { + const query = args[0] as { sql: string }, + result = process(...args) + if (query.sql.includes('ORDINAL_POSITION position')) { + altered = true + if (kind === 'nonarray') return [null] + if (kind === '513 rows') + return [Array.from({ length: 513 }, (_, i) => ({ ...result[0][0], name: 'column_' + i }))] + result[0][0].defaultValue = kind === 'multibyte value' ? '😀'.repeat(4097) : 'x'.repeat(16385) + } + return result + }) + try { + await expect(readSnapshotJournalMysqlBinding(k)).rejects.toThrow( + 'Unsupported or incomplete MySQL snapshot journal source' + ) + expect(altered).toBe(true) + } finally { + await k.destroy() + } + } +) + +test('binding accepts exactly 512 driver rows and one 16384-byte definition', async () => { + const k = await metadata(), + process = jest.mocked(k.client.processResponse).getMockImplementation()! + let altered = false + jest.mocked(k.client.processResponse).mockImplementation((...args: unknown[]) => { + const query = args[0] as { sql: string }, + result = process(...args) + if (query.sql.includes('ORDINAL_POSITION position')) { + altered = true + while (result[0].length < 512) result[0].push({ name: 'extra_' + result[0].length }) + result[0][0].defaultValue = '😀'.repeat(4096) + } + return result + }) + try { + expect(await readSnapshotJournalMysqlBinding(k)).toMatch(/^[0-9a-f]{64}$/) + expect(altered).toBe(true) + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlSource.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlSource.ts new file mode 100644 index 000000000..231ffcd43 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalMysqlSource.ts @@ -0,0 +1,262 @@ +import type { Knex } from 'knex' +import { createHash } from 'node:crypto' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { numeric } from '../../schema/snapshotSqliteMembership' +import { snapshotGlobalIndexTriggers } from '../../schema/snapshotGlobalIndexTriggers' +import { readSnapshotGlobalIndexState } from '../../schema/snapshotGlobalIndexMigration' +import { + snapshotProfileTriggerDefinition, + readSnapshotProfileIndexState +} from '../../schema/snapshotProfileIndexMigration' +import { + snapshotRelationIndexTriggers, + readSnapshotRelationIndexState +} from '../../schema/snapshotRelationIndexMigration' +import { + snapshotCertificateIndexTriggers, + readSnapshotCertificateIndexState +} from '../../schema/snapshotCertificateIndexMigration' + +function invalid(): never { + throw new WERR_INVALID_OPERATION('Unsupported or incomplete MySQL snapshot journal source') +} +interface SourceColumn { + tableName: string + name: string + type: string + nullable: string + extra: string +} +function validateNumericColumns(columns: SourceColumn[]): void { + for (const source of numeric) { + for (const name of [source.key, ...(source.owner ? [source.owner] : [])]) { + const found = columns.filter(column => column.tableName === source.table && column.name === name) + if ( + found.length !== 1 || + found[0].type.replaceAll(/\(\d+\)/g, '') !== 'int unsigned' || + found[0].nullable !== 'NO' || + !(found[0].extra === '' || (name === source.key && found[0].extra === 'auto_increment')) + ) + return invalid() + } + } +} +/** Read-only prerequisite guard; ownership/install intents and epoch/receipt provenance remain the migrator's responsibility. */ +export async function validateSnapshotJournalMysqlSource(k: Knex, config?: Knex.MigratorConfig): Promise { + if (!['mysql', 'mysql2'].includes(k.client.config.client)) return invalid() + await runInSeries( + [ + readSnapshotProfileIndexState, + readSnapshotRelationIndexState, + readSnapshotCertificateIndexState, + readSnapshotGlobalIndexState + ], + async read => { + if (!(await read(k, config))) invalid() + } + ) + const sourceTables = [ + ...numeric.map(source => source.table), + 'tx_labels_map', + 'output_tags_map', + 'certificate_fields' + ] + const [tables]: Array> = await k.raw( + 'SELECT TABLE_NAME name,ENGINE engine,TABLE_TYPE type FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (' + + sourceTables.map(() => '?').join(',') + + ')', + sourceTables + ) + if ( + tables.length !== sourceTables.length || + tables.some(table => table.engine !== 'InnoDB' || table.type !== 'BASE TABLE') + ) + return invalid() + const [columns]: Array = await k.raw( + 'SELECT TABLE_NAME tableName,COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,EXTRA extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (' + + sourceTables.map(() => '?').join(',') + + ') ORDER BY TABLE_NAME,ORDINAL_POSITION LIMIT 513', + sourceTables + ) + const [parts]: Array< + Array<{ + tableName: string + name: string + columnName: string + nonUnique: number + direction: string + prefix: unknown + visible: string + }> + > = await k.raw( + 'SELECT TABLE_NAME tableName,INDEX_NAME name,COLUMN_NAME columnName,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,IS_VISIBLE visible FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (' + + sourceTables.map(() => '?').join(',') + + ') ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX LIMIT 513', + sourceTables + ) + if (columns.length > 512 || parts.length > 512) return invalid() + validateNumericColumns(columns) + const identities = [ + ...numeric.map(source => ({ table: source.table, keys: [source.key], primary: true })), + { table: 'tx_labels_map', keys: ['txLabelId', 'transactionId'], primary: false }, + { table: 'output_tags_map', keys: ['outputTagId', 'outputId'], primary: false }, + { table: 'certificate_fields', keys: ['fieldName', 'certificateId'], primary: false } + ] + for (const source of identities) { + const candidates = parts.filter(part => part.tableName === source.table && part.nonUnique === 0) + const matches = [...new Set(candidates.map(part => part.name))].some(name => { + const index = candidates.filter(part => part.name === name) + return ( + (!source.primary || name === 'PRIMARY') && + index.length === source.keys.length && + index.every( + (part, i) => + part.columnName === source.keys[i] && + part.direction === 'A' && + part.prefix === null && + part.visible === 'YES' + ) + ) + }) + if (!matches) return invalid() + } + const [rules]: Array> = await k.raw( + 'SELECT UPDATE_RULE updateRule,DELETE_RULE deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME IN (' + + sourceTables.map(() => '?').join(',') + + ')', + sourceTables + ) + if ( + rules.some( + rule => ![rule.updateRule, rule.deleteRule].every(value => value === 'RESTRICT' || value === 'NO ACTION') + ) + ) + return invalid() + const profile = numeric + .filter(source => source.owner !== undefined) + .flatMap((source, tableId) => + (['INSERT', 'UPDATE', 'DELETE'] as const).map(event => ({ + ...snapshotProfileTriggerDefinition(true, source.table, source.key, tableId, event), + table: source.table, + event, + timing: 'AFTER' + })) + ) + const relation = snapshotRelationIndexTriggers(true), + certificate = snapshotCertificateIndexTriggers(true) + const expected = [ + ...profile, + ...relation.observers, + ...relation.producers, + ...certificate.observers, + ...certificate.producers, + ...snapshotGlobalIndexTriggers(true) + ] + const maximum = Math.max(...expected.map(trigger => Buffer.byteLength(trigger.body, 'utf8'))) + 1 + const [actual]: Array> = + await k.raw( + 'SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME IN (' + + expected.map(() => '?').join(',') + + ')', + [maximum, ...expected.map(trigger => trigger.name)] + ) + if (actual.length !== expected.length) return invalid() + for (const trigger of expected) { + const found = actual.filter(row => row.name === trigger.name) + if ( + found.length !== 1 || + found[0].table !== trigger.table || + found[0].event !== trigger.event || + found[0].timing !== trigger.timing || + found[0].body !== trigger.body + ) + return invalid() + } +} + +/** Bind static source semantics, never row counts, statistics or AUTO_INCREMENT positions. + * Call only with operator-owned schema changes excluded; this read is not a DDL lock. + * The epoch owner persists the returned digest and refuses a different binding. + */ +export async function readSnapshotJournalMysqlBinding(k: Knex, config?: Knex.MigratorConfig): Promise { + await validateSnapshotJournalMysqlSource(k, config) + const tables = [ + ...numeric.map(source => source.table), + 'tx_labels_map', + 'output_tags_map', + 'certificate_fields', + 'users', + 'settings', + 'snapshot_profile_keys', + 'snapshot_relation_keys', + 'snapshot_certificate_field_keys', + 'snapshot_global_edges', + 'snapshot_global_keys', + 'snapshot_global_guards' + ] + const marks = tables.map(() => '?').join(',') + const journalNames = [ + ...Array.from({ length: 4 }, (_, i) => 'snapshot_journal_scope_' + i), + ...Array.from({ length: 13 }, (_, i) => 'snapshot_journal_physical_' + i) + ].flatMap(prefix => ['INSERT', 'UPDATE', 'DELETE'].map(event => prefix + '_' + event)) + // Hard query cardinality and value limits keep metadata capture independent of wallet size. + // Reject truncation rather than hashing a prefix as if it described the whole schema. + const bound = (column: string) => `SUBSTRING(${column},1,16385)` + const definitions = [ + [ + 'tables', + `SELECT TABLE_NAME name,ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,${bound('CREATE_OPTIONS')} options FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (${marks}) ORDER BY TABLE_NAME` + ], + [ + 'columns', + `SELECT TABLE_NAME tableName,COLUMN_NAME name,ORDINAL_POSITION position,COLUMN_TYPE type,IS_NULLABLE nullable,${bound('COLUMN_DEFAULT')} defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,${bound('GENERATION_EXPRESSION')} expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (${marks}) ORDER BY TABLE_NAME,ORDINAL_POSITION` + ], + [ + 'indexes', + `SELECT TABLE_NAME tableName,INDEX_NAME name,SEQ_IN_INDEX position,COLUMN_NAME columnName,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,NULLABLE nullable,INDEX_TYPE type,IS_VISIBLE visible,${bound('EXPRESSION')} expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (${marks}) ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX` + ], + [ + 'foreignKeys', + `SELECT TABLE_NAME tableName,CONSTRAINT_NAME name,ORDINAL_POSITION position,COLUMN_NAME columnName,REFERENCED_TABLE_SCHEMA foreignSchema,REFERENCED_TABLE_NAME foreignTable,REFERENCED_COLUMN_NAME foreignColumn FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (${marks}) AND REFERENCED_TABLE_NAME IS NOT NULL ORDER BY TABLE_NAME,CONSTRAINT_NAME,ORDINAL_POSITION` + ], + [ + 'foreignRules', + `SELECT TABLE_NAME tableName,CONSTRAINT_NAME name,MATCH_OPTION matchOption,UPDATE_RULE updateRule,DELETE_RULE deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME IN (${marks}) ORDER BY TABLE_NAME,CONSTRAINT_NAME` + ], + [ + 'checks', + `SELECT t.TABLE_NAME tableName,t.CONSTRAINT_NAME name,t.ENFORCED enforced,${bound('c.CHECK_CLAUSE')} clause FROM information_schema.TABLE_CONSTRAINTS t JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME IN (${marks}) AND t.CONSTRAINT_TYPE='CHECK' ORDER BY t.TABLE_NAME,t.CONSTRAINT_NAME` + ], + [ + 'triggers', + `SELECT EVENT_OBJECT_TABLE tableName,TRIGGER_NAME name,EVENT_MANIPULATION event,ACTION_TIMING timing,ACTION_ORDER actionOrder,DEFINER definer,${bound('ACTION_STATEMENT')} body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION connectionCollation,DATABASE_COLLATION databaseCollation FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE IN (${marks}) AND TRIGGER_NAME NOT IN (${journalNames.map(() => '?').join(',')}) ORDER BY EVENT_OBJECT_TABLE,TRIGGER_NAME` + ] + ] as const + const digest = createHash('sha256').update('snapshot-journal-mysql-binding-v1\n') + let bytes = 0 + await runInSeries(definitions, async ([kind, sql]) => { + const [rows]: Array>> = await k.raw( + sql + ' LIMIT 513', + kind === 'triggers' ? [...tables, ...journalNames] : tables + ) + if ( + !Array.isArray(rows) || + rows.length > 512 || + rows.some(row => + Object.values(row).some(value => typeof value === 'string' && Buffer.byteLength(value, 'utf8') > 16384) + ) + ) + invalid() + if ( + kind === 'tables' && + (rows.length !== tables.length || rows.some(row => row.type !== 'BASE TABLE' || row.engine !== 'InnoDB')) + ) + invalid() + const canonical = JSON.stringify([kind, rows]) + bytes += Buffer.byteLength(canonical, 'utf8') + if (bytes > 1048576) invalid() + digest.update(canonical) + }) + return digest.digest('hex') +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalPage.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalPage.ts new file mode 100644 index 000000000..68b99b7e0 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalPage.ts @@ -0,0 +1,199 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { + compareSnapshotJournalRevisions, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' +import { snapshotJournalRevisionOperand, snapshotJournalRevisionText } from './SnapshotJournalRevisionSql' + +export interface SnapshotJournalPosition { + revision: SnapshotJournalRevision + id1: number + id2: number + exactText: string +} + +export interface SnapshotJournalInterval { + stream: 'scope' | 'physical' + tableId: number + userId: number + floor: SnapshotJournalRevision + low: SnapshotJournalRevision + high: SnapshotJournalRevision + after?: SnapshotJournalPosition + limit: number +} + +export interface SnapshotJournalMetadata extends SnapshotJournalPosition { + present: boolean + generation?: SnapshotJournalRevision +} + +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid snapshot journal interval or metadata') +} + +function integer(value: unknown, minimum: number, maximum = Number.MAX_SAFE_INTEGER): number { + if (typeof value !== 'number' || !Number.isSafeInteger(value) || value < minimum || value > maximum) invalid() + return value +} + +/** Source keys are exact UTF-8 bytes, independent of the source's text collation. */ +function textKey(value: unknown): string { + const text = value instanceof Uint8Array ? Buffer.from(value).toString('utf8') : value + if (typeof text !== 'string') invalid() + const bytes = Buffer.from(text, 'utf8') + if (bytes.byteLength > 400 || bytes.toString('utf8') !== text) invalid() + if (value instanceof Uint8Array && !bytes.equals(Buffer.from(value))) invalid() + return text +} + +function sqlite(k: Knex): boolean { + const client = k.client.config.client + if (client === 'sqlite3' || client === 'better-sqlite3') return true + if (client === 'mysql' || client === 'mysql2') return false + throw new WERR_INVALID_OPERATION('Unsupported snapshot journal SQL driver') +} + +function compare(left: SnapshotJournalPosition, right: SnapshotJournalPosition): number { + return ( + compareSnapshotJournalRevisions(left.revision, right.revision) || + left.id1 - right.id1 || + left.id2 - right.id2 || + Buffer.compare(Buffer.from(left.exactText, 'utf8'), Buffer.from(right.exactText, 'utf8')) + ) +} + +function validate(interval: SnapshotJournalInterval): void { + const { stream, tableId, userId, floor, low, high, after, limit } = interval + if (stream !== 'scope' && stream !== 'physical') invalid() + integer(tableId, 0, 12) + integer(userId, 1) + integer(limit, 1, 256) + for (const value of [floor, low, high]) snapshotJournalRevision(value) + if (compareSnapshotJournalRevisions(floor, low) > 0 || compareSnapshotJournalRevisions(low, high) > 0) invalid() + if (stream === 'physical' && tableId !== 8 && tableId !== 9) invalid() + if (after !== undefined) { + snapshotJournalRevision(after.revision) + if ( + compareSnapshotJournalRevisions(after.revision, low) <= 0 || + compareSnapshotJournalRevisions(after.revision, high) > 0 + ) + invalid() + integer(after.id1, 1) + integer(after.id2, 0) + if (typeof after.exactText !== 'string') invalid() + textKey(after.exactText) + } +} + +/** Caller supplies the same pinned connection used for floor, high-water and payload reads. */ +export function snapshotJournalMetadataQuery(k: Knex, interval: SnapshotJournalInterval): Knex.QueryBuilder { + validate(interval) + const local = sqlite(k) + const table = 'snapshot_journal_' + interval.stream + const query = k + .from(k.raw(local ? '?? AS ?? INDEXED BY ??' : '?? AS ?? FORCE INDEX (??)', [table, 'j', table + '_page'])) + .select('j.id1', 'j.id2', 'j.exactText', 'j.present') + .select({ revisionText: snapshotJournalRevisionText(k, 'j.revision') }) + .where('j.tableId', interval.tableId) + .where('j.revision', '<=', snapshotJournalRevisionOperand(k, interval.high)) + .orderBy(['j.revision', 'j.id1', 'j.id2', 'j.exactText']) + .limit(interval.limit) + if (interval.stream === 'scope') query.where('j.userId', interval.userId) + else query.select({ generationText: snapshotJournalRevisionText(k, 'j.generation') }) + const after = interval.after + if (after === undefined) query.where('j.revision', '>', snapshotJournalRevisionOperand(k, interval.low)) + else { + const revision = snapshotJournalRevisionOperand(k, after.revision) + if (local) { + query.whereRaw('(??,??,??,??) > (?,?,?,?)', [ + 'j.revision', + 'j.id1', + 'j.id2', + 'j.exactText', + revision, + after.id1, + after.id2, + after.exactText + ]) + } else { + query.where('j.revision', '>=', revision).where(function () { + this.where('j.revision', '>', revision) + .orWhere(function () { + this.where('j.revision', revision).where('j.id1', '>', after.id1) + }) + .orWhere(function () { + this.where('j.revision', revision).where('j.id1', after.id1).where('j.id2', '>', after.id2) + }) + .orWhere(function () { + this.where('j.revision', revision) + .where('j.id1', after.id1) + .where('j.id2', after.id2) + .where('j.exactText', '>', Buffer.from(after.exactText, 'utf8')) + }) + }) + } + } + return query +} + +/** The bound applies before membership filtering, including unrelated global changes. */ +export async function readSnapshotJournalMetadataPage( + k: Knex, + interval: SnapshotJournalInterval +): Promise<{ + rows: SnapshotJournalMetadata[] + examined: number + complete: boolean + after?: SnapshotJournalPosition +}> { + const query = snapshotJournalMetadataQuery(k, interval) + if (interval.low === interval.high) return { rows: [], examined: 0, complete: true } + const records: Array> = await query + if (records.length > interval.limit) invalid() + const rows = records.map(record => { + const revision = snapshotJournalRevision(record.revisionText) + if ( + compareSnapshotJournalRevisions(revision, interval.low) <= 0 || + compareSnapshotJournalRevisions(revision, interval.high) > 0 + ) + invalid() + if (![0, 1, false, true].includes(record.present as number | boolean)) invalid() + const row: SnapshotJournalMetadata = { + revision, + id1: integer(record.id1, 1), + id2: integer(record.id2, 0), + exactText: textKey(record.exactText), + present: record.present === 1 || record.present === true + } + if (interval.stream === 'physical') { + const generation = snapshotJournalRevision(record.generationText) + if (generation === '0' || compareSnapshotJournalRevisions(generation, revision) > 0) invalid() + row.generation = generation + } + return row + }) + let previous = interval.after + for (const row of rows) { + if (previous !== undefined && compare(previous, row) >= 0) invalid() + previous = row + } + const last = rows.at(-1) + return { + rows, + examined: rows.length, + complete: rows.length < interval.limit, + ...(last === undefined + ? {} + : { + after: { + revision: last.revision, + id1: last.id1, + id2: last.id2, + exactText: last.exactText + } + }) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalPrerequisites.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalPrerequisites.test.ts new file mode 100644 index 000000000..cd9cd8f41 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalPrerequisites.test.ts @@ -0,0 +1,54 @@ +import { + readSnapshotJournalSqliteGeneration, + completeSnapshotJournalSqliteGeneration +} from './SnapshotJournalSqliteGeneration' +import { knex, type Knex } from 'knex' +import { copySnapshotJournalBootstrapPage } from './SnapshotJournalBootstrap' +import { readSnapshotJournalHighWater } from './SnapshotJournalHighWater' +import { snapshotJournalSqliteObserverSql } from './SnapshotJournalSqliteObservers' +import { snapshotJournalMysqlObserverSql } from './SnapshotJournalMysqlObservers' +import { snapshotJournalRevision as rev } from './SnapshotJournalRevision' + +const journalReceiptPolicy = { receiptLimit: 128, receiptLifetimeMs: 2592000000 } + +test.each([ + ['bootstrap', (k: Knex) => copySnapshotJournalBootstrapPage(k, 1000000)], + ['SQLite generation read', (k: Knex) => readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)], + ['SQLite generation completion', (k: Knex) => completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)], + ['high-water', (k: Knex) => readSnapshotJournalHighWater(k, 1, rev('0'), rev('0'))], + ['SQLite observers', (k: Knex) => snapshotJournalSqliteObserverSql(k)], + ['MySQL observers', (k: Knex) => snapshotJournalMysqlObserverSql(k)] +] as const)('%s rejects an unsupported driver before I/O', async (_name, run) => { + const k = knex({ client: 'mysql2' }), + query = jest.fn() + k.client.config.client = 'unsupported' + k.on('query', query) + try { + await expect(run(k)).rejects.toThrow() + expect(query).not.toHaveBeenCalled() + } finally { + await k.destroy() + } +}) +test('SQLite observer preparation refuses a missing source without any schema writes', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }), + queries: string[] = [] + k.on('query', q => queries.push(q.sql)) + try { + await expect(snapshotJournalSqliteObserverSql(k)).rejects.toThrow('Missing snapshot journal source') + expect(queries).toEqual(['PRAGMA table_info(`transactions`)']) + expect(await k('sqlite_master').select('name')).toEqual([]) + } finally { + await k.destroy() + } +}) +test('MySQL observer preparation refuses missing driver metadata without emitting DDL', async () => { + const raw = jest.fn(async () => [[]]), + k = { client: { config: { client: 'mysql2' } }, raw } as unknown as Knex + await expect(snapshotJournalMysqlObserverSql(k)).rejects.toThrow('Missing snapshot journal source') + expect(raw).toHaveBeenCalledTimes(1) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceipt.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceipt.test.ts new file mode 100644 index 000000000..cfc5f1abd --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceipt.test.ts @@ -0,0 +1,592 @@ +import { knex, type Knex } from 'knex' +import { createHash } from 'node:crypto' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL } from './SnapshotJournalSqliteClock' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import * as ArchiveClock from '../archive/SnapshotArchiveSql' +import { + advanceSnapshotJournalFloor, + snapshotJournalReceiptPolicy, + snapshotJournalReceiptBinding, + snapshotJournalReceiptDdl, + recordSnapshotJournalReceipt, + readSnapshotJournalReceipt, + collectSnapshotJournalReceipts, + type SnapshotJournalReceiptBinding +} from './SnapshotJournalReceipt' +import { snapshotJournalRevision } from './SnapshotJournalRevision' + +const binding: SnapshotJournalReceiptBinding = { + backend: '11'.repeat(32), + epoch: '12345678-1234-4234-9234-123456789012', + source: '22'.repeat(32), + schema: '33'.repeat(32), + storageIdentity: 'synthetic-storage', + identityKey: '02' + '44'.repeat(32), + userId: 1, + chain: 'test' +} +const request = (n: number) => ({ + requestId: n.toString(16).padStart(64, '0'), + highWater: snapshotJournalRevision('9007199254740993'), + expiresAt: Date.now() + 60000 +}) +let k: Knex +beforeEach(async () => { + k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + await k.raw('PRAGMA busy_timeout=0') + for (const sql of snapshotJournalReceiptDdl(k)) await k.raw(sql) + await k('snapshot_journal_retention').insert({ id: 1, floor: '0', receiptLimit: 128, receiptLifetimeMs: 2592000000 }) +}) +afterEach(async () => { + await k.destroy() +}) + +test('binding has a stable independent digest and distinguishes every bound field', () => { + const expected = createHash('sha256') + .update( + 'snapshot-journal-receipt-binding-v1\n' + + JSON.stringify([ + binding.backend, + binding.epoch, + binding.source, + binding.schema, + binding.storageIdentity, + binding.identityKey, + binding.userId, + binding.chain + ]) + ) + .digest('hex') + expect(snapshotJournalReceiptBinding(binding)).toBe(expected) + const changes: Partial[] = [ + { backend: '55'.repeat(32) }, + { epoch: '12345678-1234-4234-9234-123456789013' }, + { source: '55'.repeat(32) }, + { schema: '55'.repeat(32) }, + { storageIdentity: 'synthetic-other' }, + { identityKey: '03' + '44'.repeat(32) }, + { userId: 2 }, + { chain: 'main' } + ] + for (const change of changes) expect(snapshotJournalReceiptBinding({ ...binding, ...change })).not.toBe(expected) +}) + +test.each([ + { backend: 'z'.repeat(64) }, + { epoch: '12345678-1234-1234-9234-123456789012' }, + { source: '11'.repeat(33) }, + { schema: '' }, + { identityKey: '04' + '44'.repeat(32) }, + { userId: 0 }, + { userId: 1.5 }, + { userId: Number.MAX_SAFE_INTEGER + 1 }, + { chain: 'unknown' }, + { storageIdentity: '' }, + { storageIdentity: 'x'.repeat(257) }, + { storageIdentity: '\ud800' }, + { storageIdentity: 'é'.repeat(129) } +])('rejects invalid binding %o before creating a receipt', async change => { + await expect( + k.transaction(t => recordSnapshotJournalReceipt(t, { ...binding, ...change }, request(1))) + ).rejects.toThrow() + expect(await k('snapshot_journal_receipts')).toHaveLength(0) +}) + +test('exact retry preserves immutable receipt/floor and binding changes refuse', async () => { + const input = request(1) + const first = await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + expect(first).toEqual({ ...input, binding: snapshotJournalReceiptBinding(binding), floor: '0' }) + await k('snapshot_journal_retention').update({ floor: '5' }) + const retry = await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + expect(retry).toEqual(first) + expect(await k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).toEqual(first) + for (const other of [ + { ...input, highWater: snapshotJournalRevision('9007199254740994') }, + { ...input, expiresAt: input.expiresAt + 1 } + ]) + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, other))).rejects.toThrow() + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, { ...binding, userId: 2 }, input))).rejects.toThrow() + await expect( + k.transaction(t => readSnapshotJournalReceipt(t, { ...binding, backend: '55'.repeat(32) }, input)) + ).rejects.toThrow() + expect(await k('snapshot_journal_receipts')).toHaveLength(1) +}) + +test('receipt and floor changes roll back together; committed receipt survives lost acknowledgement', async () => { + const input = request(1) + await expect( + k.transaction(async t => { + await t('snapshot_journal_retention').update({ floor: '7' }) + await recordSnapshotJournalReceipt(t, binding, input) + throw new Error('before commit') + }) + ).rejects.toThrow('before commit') + expect(await k('snapshot_journal_receipts')).toHaveLength(0) + expect((await k('snapshot_journal_retention').first()).floor).toBe('0') + await expect( + (async () => { + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + throw new Error('after commit') + })() + ).rejects.toThrow('after commit') + expect(await k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).toMatchObject(input) +}) + +test('a missing restored receipt refuses even after journal revision moves beyond the checkpoint', async () => { + const input = request(1) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + const exact = await k('snapshot_journal_receipts').first() + await k('snapshot_journal_receipts').delete() + await k.transaction(t => + recordSnapshotJournalReceipt(t, binding, { ...request(2), highWater: snapshotJournalRevision('9007199254740994') }) + ) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow() + await k('snapshot_journal_receipts').insert(exact) + expect(await k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).toMatchObject(input) + await k('snapshot_journal_retention').update({ floor: '9007199254740994' }) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow() +}) + +test('capacity includes expired records until bounded collection commits, and exact expired requests cannot reopen', async () => { + await k('snapshot_journal_retention').update({ receiptLimit: 2 }) + const a = request(1), + b = request(2), + c = request(3) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, b)) + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, c))).rejects.toThrow() + await k('snapshot_journal_receipts').where('requestId', a.requestId).update({ expiresAt: 1 }) + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, c))).rejects.toThrow() + await expect( + k.transaction(async t => { + expect(await collectSnapshotJournalReceipts(t)).toBe(1) + throw new Error('collector rollback') + }) + ).rejects.toThrow('collector rollback') + expect(await k('snapshot_journal_receipts')).toHaveLength(2) + expect(await k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(1) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, c)) + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, { ...a, expiresAt: 1 }))).rejects.toThrow() + expect(await k.transaction(t => readSnapshotJournalReceipt(t, binding, b))).toMatchObject(b) +}) + +test('collector uses at most64 indexed candidates and releases no unexpired receipt', async () => { + const common = { binding: snapshotJournalReceiptBinding(binding), highWater: '9', floor: '0', expiresAt: 1 } + for (let i = 1; i <= 128; i += 32) + await k('snapshot_journal_receipts').insert( + Array.from({ length: 32 }, (_, offset) => ({ ...common, requestId: (i + offset).toString(16).padStart(64, '0') })) + ) + await k('snapshot_journal_receipts') + .where('requestId', request(128).requestId) + .update({ expiresAt: Date.now() + 60000 }) + const query = k('snapshot_journal_receipts') + .where('expiresAt', '<=', Date.now()) + .select('requestId') + .orderBy(['expiresAt', 'requestId']) + .limit(64) + .toSQL() + const plan: Array<{ detail: string }> = await k.raw( + 'EXPLAIN QUERY PLAN ' + query.sql, + query.bindings as Knex.RawBinding[] + ) + expect(plan.map(x => x.detail).join(' ')).toContain( + 'SEARCH snapshot_journal_receipts USING COVERING INDEX snapshot_journal_receipts_expiry' + ) + expect(await k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(64) + expect(await k('snapshot_journal_receipts')).toHaveLength(64) + expect(await k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(63) + expect(await k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(0) + expect(await k('snapshot_journal_receipts')).toHaveLength(1) +}) + +test('mutating calls require explicit transactions and a nonwaiting SQLite connection', async () => { + await expect(recordSnapshotJournalReceipt(k, binding, request(1))).rejects.toThrow() + await expect(readSnapshotJournalReceipt(k, binding, request(1))).rejects.toThrow() + await expect(collectSnapshotJournalReceipts(k)).rejects.toThrow() + await k.raw('PRAGMA busy_timeout=1') + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, request(1)))).rejects.toThrow() + await expect(k.transaction(t => collectSnapshotJournalReceipts(t))).rejects.toThrow() +}) + +test('malformed persisted receipts and retention metadata refuse', async () => { + const input = request(1) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + for (const change of [{ binding: 'x' }, { highWater: '01' }, { floor: '9007199254740994' }, { expiresAt: 1 }]) { + await k.transaction(async t => { + await t('snapshot_journal_receipts').update(change) + await expect(readSnapshotJournalReceipt(t, binding, input)).rejects.toThrow() + await t.rollback() + }) + } + await k('snapshot_journal_retention').update({ floor: '01' }) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow() +}) + +test('expired, overlong and malformed requests allocate no receipt', async () => { + const input = request(1) + for (const change of [ + { requestId: 'x' }, + { requestId: 'a'.repeat(65) }, + { highWater: '0' }, + { highWater: '01' }, + { expiresAt: 1 }, + { expiresAt: Date.now() + 2592000000 + 60000 }, + { expiresAt: '9007199254740993' } + ]) + await expect( + k.transaction(t => recordSnapshotJournalReceipt(t, binding, { ...input, ...change } as typeof input)) + ).rejects.toThrow() + expect(await k('snapshot_journal_receipts')).toHaveLength(0) +}) + +test('a receipt whose captured floor exceeds the current floor refuses inconsistent state', async () => { + const input = request(1) + await k('snapshot_journal_retention').update({ floor: '10' }) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + await k('snapshot_journal_retention').update({ floor: '9' }) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow() + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, input))).rejects.toThrow() +}) + +test('policy rejects non-record inputs with the stable receipt error identity', () => { + const fields = { receiptLimit: 1, receiptLifetimeMs: 1 } + const malformed: unknown[] = [null, undefined, 1, 'policy', Object.assign([], fields), Object.assign(() => 0, fields)] + for (const value of malformed) { + const call = () => snapshotJournalReceiptPolicy(value as typeof fields) + expect(call).toThrow(WERR_INVALID_OPERATION) + expect(call).toThrow('Invalid, unavailable or expired snapshot journal receipt') + } + expect(snapshotJournalReceiptPolicy(fields)).toEqual(fields) + expect(snapshotJournalReceiptPolicy(fields)).not.toBe(fields) + expect(snapshotJournalReceiptPolicy({ receiptLimit: 128, receiptLifetimeMs: 2592000000 })).toEqual({ + receiptLimit: 128, + receiptLifetimeMs: 2592000000 + }) +}) + +test('binding refuses non-record values and textual lookalikes before hashing', () => { + for (const value of [null, undefined, 1, 'binding', Object.assign(() => 0, binding)]) + expect(() => snapshotJournalReceiptBinding(value as unknown as SnapshotJournalReceiptBinding)).toThrow( + WERR_INVALID_OPERATION + ) + for (const field of ['backend', 'source', 'schema', 'epoch', 'identityKey', 'storageIdentity'] as const) { + const text = binding[field] + expect(() => snapshotJournalReceiptBinding({ ...binding, [field]: { toString: () => text } })).toThrow( + WERR_INVALID_OPERATION + ) + } + for (const field of ['epoch', 'identityKey'] as const) + for (const text of ['x' + binding[field], binding[field] + 'x']) + expect(() => snapshotJournalReceiptBinding({ ...binding, [field]: text })).toThrow(WERR_INVALID_OPERATION) +}) + +test('all supported chains and exact UTF-8 storage identity boundaries bind distinctly', () => { + const chains = ['main', 'test', 'stn', 'ttn', 'tstn', 'mock'] + expect(new Set(chains.map(chain => snapshotJournalReceiptBinding({ ...binding, chain }))).size).toBe(chains.length) + const identities = ['x', 'x'.repeat(256), 'é'.repeat(128), '😀'.repeat(64)] + const hashes = identities.map(storageIdentity => snapshotJournalReceiptBinding({ ...binding, storageIdentity })) + expect(new Set(hashes).size).toBe(identities.length) + for (const hash of hashes) expect(hash).toMatch(/^[0-9a-f]{64}$/) +}) + +test('exact floor and lifetime boundaries remain readable until the database expiry instant', async () => { + let time = 1000000 + const clock = jest.spyOn(ArchiveClock, 'snapshotArchiveDatabaseNow').mockImplementation(async () => time) + try { + const input = { ...request(1), highWater: snapshotJournalRevision('7'), expiresAt: time + 1000 } + await k('snapshot_journal_retention').update({ floor: '7', receiptLifetimeMs: 1000 }) + const expected = { ...input, binding: snapshotJournalReceiptBinding(binding), floor: '7' } + expect(await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input))).toEqual(expected) + expect(await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input))).toEqual(expected) + expect(await k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).toEqual(expected) + for (const changed of [ + { ...input, highWater: snapshotJournalRevision('8') }, + { ...input, expiresAt: input.expiresAt + 1 } + ]) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, changed))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + time = input.expiresAt - 1 + expect(await k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).toEqual(expected) + expect(await k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(0) + time = input.expiresAt + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + expect(await k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(1) + expect(await k('snapshot_journal_receipts')).toEqual([]) + } finally { + clock.mockRestore() + } +}) + +test.each([0, -1, 1.5, Number.NaN, Number.POSITIVE_INFINITY, Number.MAX_SAFE_INTEGER + 1])( + 'invalid database time %s refuses allocation, lookup and collection without changing rows', + async time => { + const input = request(1) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + const before = await k('snapshot_journal_receipts') + const clock = jest.spyOn(ArchiveClock, 'snapshotArchiveDatabaseNow').mockResolvedValue(time) + try { + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, request(2)))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + await expect(k.transaction(t => collectSnapshotJournalReceipts(t))).rejects.toThrow(WERR_INVALID_OPERATION) + expect(await k('snapshot_journal_receipts')).toEqual(before) + } finally { + clock.mockRestore() + } + } +) + +test('malformed singleton retention metadata refuses every operation without partial writes', async () => { + const input = request(1) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + const retained = await k('snapshot_journal_retention').first() + const before = await k('snapshot_journal_receipts') + await k.raw('PRAGMA ignore_check_constraints=ON') + const malformed = [ + [], + [{ ...retained, id: 2 }], + [retained, { ...retained, id: 2 }], + [{ ...retained, receiptLimit: 0 }], + [{ ...retained, receiptLimit: 129 }], + [{ ...retained, receiptLifetimeMs: 0 }], + [{ ...retained, receiptLifetimeMs: 2592000001 }] + ] + for (const rows of malformed) { + await k('snapshot_journal_retention').delete() + if (rows.length) await k('snapshot_journal_retention').insert(rows) + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, request(2)))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + await expect(k.transaction(t => collectSnapshotJournalReceipts(t))).rejects.toThrow(WERR_INVALID_OPERATION) + expect(await k('snapshot_journal_receipts')).toEqual(before) + } +}) + +test('malformed stored receipt revisions and collector identities refuse without deleting data', async () => { + const input = request(1) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input)) + const stored = await k('snapshot_journal_receipts').first() + for (const highWater of ['0', '-1', '9223372036854775808']) { + await k('snapshot_journal_receipts').update({ highWater }) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow() + await expect(k.transaction(t => recordSnapshotJournalReceipt(t, binding, input))).rejects.toThrow() + } + await k('snapshot_journal_receipts').update({ ...stored, requestId: 'not-a-digest', expiresAt: 1 }) + await expect(k.transaction(t => collectSnapshotJournalReceipts(t))).rejects.toThrow(WERR_INVALID_OPERATION) + expect(await k('snapshot_journal_receipts')).toHaveLength(1) +}) + +test('SQLite alias is supported while unrecognized database drivers refuse explicitly', async () => { + k.client.config.client = 'sqlite3' + const input = request(1) + expect(await k.transaction(t => recordSnapshotJournalReceipt(t, binding, input))).toMatchObject(input) + for (const client of ['pg', 'mysql-compatible', '', undefined]) { + k.client.config.client = client + expect(() => snapshotJournalReceiptDdl(k)).toThrow(WERR_INVALID_OPERATION) + await expect(k.transaction(t => readSnapshotJournalReceipt(t, binding, input))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + } +}) + +test('request expiry must be an exact primitive number before database work', async () => { + const input = request(1) + for (const expiresAt of [String(input.expiresAt), { valueOf: () => input.expiresAt }, null, undefined, Number.NaN]) + await expect( + k.transaction(t => recordSnapshotJournalReceipt(t, binding, { ...input, expiresAt } as typeof input)) + ).rejects.toThrow(WERR_INVALID_OPERATION) + expect(await k('snapshot_journal_receipts')).toEqual([]) +}) + +describe('atomic continuity floor', () => { + let directory: string + beforeEach(async () => { + await k.destroy() + directory = await mkdtemp(join(tmpdir(), 'ts569-retention-')) + k = knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + await k.raw('PRAGMA journal_mode=WAL') + await k.raw('PRAGMA busy_timeout=0') + for (const sql of snapshotJournalReceiptDdl(k)) await k.raw(sql) + await k.raw(SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL) + await k('snapshot_journal_clock').insert({ id: 1, revision: '100', ceiling: '1000', enabled: 1, reason: null }) + await k('snapshot_journal_retention').insert({ id: 1, floor: '0', receiptLimit: 128, receiptLifetimeMs: 600000 }) + jest.spyOn(ArchiveClock, 'snapshotArchiveDatabaseNow').mockResolvedValue(2000) + }) + afterEach(async () => { + jest.restoreAllMocks() + await k.destroy() + await rm(directory, { recursive: true, force: true }) + }) + const revision = (n: number) => snapshotJournalRevision(String(n)) + async function stored(n: number, highWater: number, expiresAt = 3000) { + await k('snapshot_journal_receipts').insert({ + requestId: n.toString(16).padStart(64, '0'), + binding: 'a'.repeat(64), + highWater: String(highWater), + floor: '0', + expiresAt + }) + } + test('the lowest live receipt pins continuity, including receipts for other profiles', async () => { + await stored(1, 90) + await stored(2, 70) + await stored(3, 10, 2000) + const receipts = await k('snapshot_journal_receipts').orderBy('requestId') + await expect(k.transaction(t => advanceSnapshotJournalFloor(t, revision(71)))).rejects.toThrow() + expect((await k('snapshot_journal_clock').first()).revision).toBe(100) + expect((await k('snapshot_journal_retention').first()).floor).toBe('0') + expect(await k('snapshot_journal_receipts').orderBy('requestId')).toEqual(receipts) + expect(await k.transaction(t => advanceSnapshotJournalFloor(t, revision(70)))).toEqual({ + floor: '70', + highWater: '101', + liveReceipts: 2, + examined: 3 + }) + expect(await k.transaction(t => advanceSnapshotJournalFloor(t, revision(70)))).toEqual({ + floor: '70', + highWater: '102', + liveReceipts: 2, + examined: 3 + }) + expect(await k('snapshot_journal_receipts').orderBy('requestId')).toEqual(receipts) + }) + test('rejects backwards and future floors and requires a writer transaction', async () => { + await k('snapshot_journal_retention').update({ floor: '40' }) + await expect(advanceSnapshotJournalFloor(k, revision(40))).rejects.toThrow() + for (const floor of [39, 102]) + await expect(k.transaction(t => advanceSnapshotJournalFloor(t, revision(floor)))).rejects.toThrow() + expect((await k('snapshot_journal_retention').first()).floor).toBe('40') + expect((await k('snapshot_journal_clock').first()).revision).toBe(100) + }) + test('an exhausted generation commits invalidation without advancing continuity', async () => { + await k('snapshot_journal_clock').update({ revision: 1000 }) + expect(await k.transaction(t => advanceSnapshotJournalFloor(t, revision(50)))).toBeUndefined() + expect((await k('snapshot_journal_clock').first()).enabled).toBe(0) + expect((await k('snapshot_journal_retention').first()).floor).toBe('0') + }) + test('floor publication rolls back atomically and survives a lost committed acknowledgement', async () => { + await expect( + k.transaction(async t => { + await advanceSnapshotJournalFloor(t, revision(50)) + throw Error('before commit') + }) + ).rejects.toThrow('before commit') + expect((await k('snapshot_journal_retention').first()).floor).toBe('0') + await expect( + (async () => { + await k.transaction(t => advanceSnapshotJournalFloor(t, revision(50))) + throw Error('lost acknowledgement') + })() + ).rejects.toThrow('lost acknowledgement') + expect((await k('snapshot_journal_retention').first()).floor).toBe('50') + }) + test('bounded capacity scans refuse overfull storage, including expired rows', async () => { + await k('snapshot_journal_retention').update({ receiptLimit: 1 }) + await stored(1, 10, 1000) + await stored(2, 10, 1000) + const queries: string[] = [] + k.on('query', query => queries.push(query.sql)) + await expect(k.transaction(t => advanceSnapshotJournalFloor(t, revision(50)))).rejects.toThrow() + expect(queries.filter(sql => sql.includes('from `snapshot_journal_receipts`'))).toEqual([ + expect.stringMatching(/order by `requestId` asc limit \?$/) + ]) + expect((await k('snapshot_journal_retention').first()).floor).toBe('0') + }) + test('corrupt expired receipts also prevent floor publication', async () => { + await stored(1, 10, 1000) + await k('snapshot_journal_receipts').update({ binding: 'z'.repeat(64) }) + await expect(k.transaction(t => advanceSnapshotJournalFloor(t, revision(50)))).rejects.toThrow() + expect((await k('snapshot_journal_retention').first()).floor).toBe('0') + }) + test('an independently held writer makes floor admission fail without waiting', async () => { + const peer = knex({ + client: 'better-sqlite3', + connection: k.client.config.connection, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + await peer.raw('PRAGMA busy_timeout=0') + const held = await peer.transaction() + try { + await held('snapshot_journal_clock').where('id', 1).update({ revision: 100 }) + const start = performance.now() + await expect(k.transaction(t => advanceSnapshotJournalFloor(t, revision(50)))).rejects.toMatchObject({ + code: 'SQLITE_BUSY' + }) + expect(performance.now() - start).toBeLessThan(1000) + } finally { + await held.rollback() + await peer.destroy() + } + }) + + test('300 seeded independent ledgers preserve every live prefix and monotonic floor', async () => { + let seed = 3242026 + const random = (max: number) => { + seed = (Math.imul(seed, 1664525) + 1013904223) >>> 0 + return seed % max + } + for (let n = 0; n < 300; n++) { + const prior = random(101), + requested = random(151) + const proofs = Array.from({ length: random(4) }, (_, index) => ({ + requestId: (index + 1).toString(16).padStart(64, '0'), + binding: 'a'.repeat(64), + highWater: String(random(121) + 1), + floor: '0', + expiresAt: random(2) ? 2000 : 3000 + })) + await k('snapshot_journal_receipts').delete() + await k('snapshot_journal_clock').update({ revision: 100, enabled: 1, reason: null }) + await k('snapshot_journal_retention').update({ floor: String(prior) }) + if (proofs.length) await k('snapshot_journal_receipts').insert(proofs) + const live = proofs.filter(proof => proof.expiresAt > 2000) + const allowed = + requested >= prior && + requested <= 101 && + live.every( + proof => + Number(proof.highWater) >= prior && Number(proof.highWater) <= 101 && requested <= Number(proof.highWater) + ) + const advancing = k.transaction(t => advanceSnapshotJournalFloor(t, revision(requested))) + if (allowed) { + expect(await advancing).toEqual({ + floor: String(requested), + highWater: '101', + liveReceipts: live.length, + examined: proofs.length + }) + } else { + await expect(advancing).rejects.toThrow() + } + expect((await k('snapshot_journal_retention').first()).floor).toBe(String(allowed ? requested : prior)) + expect((await k('snapshot_journal_clock').first()).revision).toBe(allowed ? 101 : 100) + expect(await k('snapshot_journal_receipts').orderBy('requestId')).toEqual(proofs) + } + }) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceipt.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceipt.ts new file mode 100644 index 000000000..c62db5b10 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceipt.ts @@ -0,0 +1,345 @@ +import type { Knex } from 'knex' +import { createHash } from 'node:crypto' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { snapshotArchiveDatabaseNow } from '../archive/SnapshotArchiveSql' +import { reserveSnapshotJournalCaptureFence } from './SnapshotJournalCaptureFence' +import { + compareSnapshotJournalRevisions, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' + +export const SNAPSHOT_JOURNAL_RECEIPT_LIMIT = 128 +export const SNAPSHOT_JOURNAL_RECEIPT_MAX_LIFETIME = 2592000000 +export interface SnapshotJournalReceiptPolicy { + receiptLimit: number + receiptLifetimeMs: number +} +export interface SnapshotJournalReceiptBinding { + backend: string + epoch: string + source: string + schema: string + storageIdentity: string + identityKey: string + userId: number + chain: string +} +export interface SnapshotJournalReceipt { + requestId: string + binding: string + highWater: SnapshotJournalRevision + floor: SnapshotJournalRevision + expiresAt: number +} +export interface SnapshotJournalRetention { + floor: SnapshotJournalRevision + receiptLimit: number + receiptLifetimeMs: number +} +const digestPattern = /^[0-9a-f]{64}$/ +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid, unavailable or expired snapshot journal receipt') +} +function local(k: Knex): boolean { + const client = k.client.config.client + if (client === 'sqlite3' || client === 'better-sqlite3') return true + if (client === 'mysql' || client === 'mysql2') return false + return invalid() +} +function transaction(k: Knex): void { + local(k) + if (!k.isTransaction) invalid() +} +const boundedInteger = (value: unknown, max: number): value is number => + typeof value === 'number' && Number.isSafeInteger(value) && value > 0 && value <= max + +function isObject(value: unknown): value is object { + return value !== null && typeof value === 'object' +} + +/** Detached installation policy; a generation cannot silently change it on resume. */ +export function snapshotJournalReceiptPolicy(value: SnapshotJournalReceiptPolicy): SnapshotJournalReceiptPolicy { + if (!isObject(value) || Array.isArray(value)) return invalid() + const { receiptLimit, receiptLifetimeMs } = value + if ( + !boundedInteger(receiptLimit, SNAPSHOT_JOURNAL_RECEIPT_LIMIT) || + !boundedInteger(receiptLifetimeMs, SNAPSHOT_JOURNAL_RECEIPT_MAX_LIFETIME) + ) + return invalid() + return { receiptLimit, receiptLifetimeMs } +} + +/** An exact, bounded digest binds a receipt without retaining profile metadata in it. */ +export function snapshotJournalReceiptBinding(value: SnapshotJournalReceiptBinding): string { + if ( + !isObject(value) || + ![value.backend, value.source, value.schema].every(part => typeof part === 'string' && digestPattern.test(part)) || + typeof value.epoch !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(value.epoch) || + typeof value.storageIdentity !== 'string' || + Buffer.byteLength(value.storageIdentity, 'utf8') < 1 || + Buffer.byteLength(value.storageIdentity, 'utf8') > 256 || + Buffer.from(value.storageIdentity, 'utf8').toString('utf8') !== value.storageIdentity || + typeof value.identityKey !== 'string' || + !/^(02|03)[0-9a-f]{64}$/.test(value.identityKey) || + !boundedInteger(value.userId, Number.MAX_SAFE_INTEGER) || + !['main', 'test', 'stn', 'ttn', 'tstn', 'mock'].includes(value.chain) + ) + return invalid() + return createHash('sha256') + .update('snapshot-journal-receipt-binding-v1\n') + .update( + JSON.stringify([ + value.backend, + value.epoch, + value.source, + value.schema, + value.storageIdentity, + value.identityKey, + value.userId, + value.chain + ]) + ) + .digest('hex') +} + +/** Owned generation DDL only. This helper neither adopts objects nor registers a migration. */ +export function snapshotJournalReceiptDdl(k: Knex): string[] { + const sqlite = local(k) + const suffix = sqlite ? '' : ' ENGINE=InnoDB DEFAULT CHARACTER SET ascii COLLATE ascii_bin ROW_FORMAT=DYNAMIC' + return [ + 'CREATE TABLE snapshot_journal_retention(id INTEGER NOT NULL PRIMARY KEY,floor VARCHAR(19) NOT NULL,receiptLimit INTEGER NOT NULL,receiptLifetimeMs BIGINT NOT NULL,CHECK(id=1),CHECK(receiptLimit BETWEEN 1 AND 128),CHECK(receiptLifetimeMs BETWEEN 1 AND 2592000000))' + + suffix, + 'CREATE TABLE snapshot_journal_receipts(requestId VARCHAR(64) NOT NULL PRIMARY KEY,binding VARCHAR(64) NOT NULL,highWater VARCHAR(19) NOT NULL,floor VARCHAR(19) NOT NULL,expiresAt BIGINT NOT NULL,CHECK(expiresAt BETWEEN 1 AND 9007199254740991)' + + (sqlite ? ')' : ',KEY snapshot_journal_receipts_expiry(expiresAt,requestId))') + + suffix, + ...(sqlite + ? ['CREATE INDEX snapshot_journal_receipts_expiry ON snapshot_journal_receipts(expiresAt,requestId)'] + : []) + ] +} +async function sharedRows(k: Knex, query: Knex.QueryBuilder): Promise>> { + if (local(k)) return await query + // Knex's MySQL forShare emits legacy LOCK IN SHARE MODE, which cannot + // take NOWAIT. Use the current-read MySQL8 form with the same bindings. + const sql = query.toSQL() + const [rows]: Array>> = await k.raw( + sql.sql + ' FOR SHARE NOWAIT', + sql.bindings as Knex.RawBinding[] + ) + return rows +} +function receiptQuery(k: Knex): Knex.QueryBuilder { + return k('snapshot_journal_receipts').select( + 'expiresAt', + ...[ + ['requestId', 65], + ['binding', 65], + ['highWater', 20], + ['floor', 20] + ].map(([field, length]) => k.raw('substr(??,1,?) AS ??', [field, length, field] as Knex.RawBinding[])) + ) +} +async function retention(k: Knex, lock: boolean): Promise { + transaction(k) + if (lock && local(k)) { + const mode: Array<{ timeout: number }> = await k.raw('PRAGMA busy_timeout') + if (mode.length !== 1 || mode[0].timeout !== 0) return invalid() + await k('snapshot_journal_retention').where('id', 1).update({ id: 1 }) + } + const query = k('snapshot_journal_retention') + .select('id', 'receiptLimit', 'receiptLifetimeMs', k.raw('substr(??,1,20) AS ??', ['floor', 'floor'])) + .limit(2) + if (lock && !local(k)) query.forUpdate().noWait() + const rows = lock ? await query : await sharedRows(k, query) + if (rows.length === 1) rows[0].receiptLifetimeMs = storedInteger(rows[0].receiptLifetimeMs) + if ( + rows.length !== 1 || + rows[0].id !== 1 || + !boundedInteger(rows[0].receiptLimit, SNAPSHOT_JOURNAL_RECEIPT_LIMIT) || + !boundedInteger(rows[0].receiptLifetimeMs, SNAPSHOT_JOURNAL_RECEIPT_MAX_LIFETIME) + ) + return invalid() + return { + floor: snapshotJournalRevision(rows[0].floor), + receiptLimit: rows[0].receiptLimit, + receiptLifetimeMs: rows[0].receiptLifetimeMs + } +} +async function now(k: Knex): Promise { + const value = await snapshotArchiveDatabaseNow(k) + if (!boundedInteger(value, Number.MAX_SAFE_INTEGER)) return invalid() + return value +} +function storedInteger(value: unknown): number { + if (typeof value === 'string' && /^(?:0|[1-9]\d{0,15})$/.test(value)) value = Number(value) + if (!boundedInteger(value, Number.MAX_SAFE_INTEGER)) return invalid() + return value +} +function receipt(value: Record, stored = false): SnapshotJournalReceipt { + if (stored) value = { ...value, expiresAt: storedInteger(value.expiresAt) } + if ( + typeof value.requestId !== 'string' || + !digestPattern.test(value.requestId) || + typeof value.binding !== 'string' || + !digestPattern.test(value.binding) || + !boundedInteger(value.expiresAt, Number.MAX_SAFE_INTEGER) + ) + return invalid() + const highWater = snapshotJournalRevision(value.highWater), + floor = snapshotJournalRevision(value.floor) + if (highWater === '0' || compareSnapshotJournalRevisions(highWater, floor) < 0) return invalid() + return { requestId: value.requestId, binding: value.binding, highWater, floor, expiresAt: value.expiresAt } +} + +/** Persist inside the caller's short writer-barrier transaction, after its separately + * reserved reader has pinned a coherent view. Never publish before that transaction + * commits. A receipt is a prefix proof, not a resumable database read transaction. + * A retry may repeat this exact proof; opening another view requires another ID. */ +export async function recordSnapshotJournalReceipt( + k: Knex, + binding: SnapshotJournalReceiptBinding, + request: { requestId: string; highWater: SnapshotJournalRevision; expiresAt: number } +): Promise { + const expected = snapshotJournalReceiptBinding(binding) + const input = receipt({ ...request, binding: expected, floor: '0' }) + const state = await retention(k, true), + time = await now(k) + if ( + input.expiresAt <= time || + input.expiresAt - time > state.receiptLifetimeMs || + compareSnapshotJournalRevisions(input.highWater, state.floor) < 0 + ) + return invalid() + const priorQuery = receiptQuery(k).where('requestId', input.requestId).first() + if (!local(k)) priorQuery.forUpdate().noWait() + const prior = await priorQuery + if (prior !== undefined) { + const existing = receipt(prior, true) + if ( + existing.binding !== expected || + existing.highWater !== input.highWater || + existing.expiresAt !== input.expiresAt || + compareSnapshotJournalRevisions(existing.highWater, state.floor) < 0 || + compareSnapshotJournalRevisions(existing.floor, state.floor) > 0 + ) + return invalid() + return existing + } + // Capacity includes expired rows until a bounded collector actually commits deletion. + const occupiedQuery = k('snapshot_journal_receipts') + .select({ occupied: 1 }) + .orderBy('requestId') + .limit(state.receiptLimit + 1) + if (!local(k)) occupiedQuery.forUpdate().noWait() + const occupied = await occupiedQuery + if (occupied.length >= state.receiptLimit) return invalid() + const result = { ...input, floor: state.floor } + await k('snapshot_journal_receipts').insert(result) + return result +} + +/** Read the floor and exact receipt in one coherent transaction. */ +export async function readSnapshotJournalReceipt( + k: Knex, + binding: SnapshotJournalReceiptBinding, + expected: { requestId: string; highWater: SnapshotJournalRevision; expiresAt: number } +): Promise { + const bound = snapshotJournalReceiptBinding(binding) + const requested = receipt({ ...expected, binding: bound, floor: '0' }) + const state = await retention(k, false), + time = await now(k) + const stored = (await sharedRows(k, receiptQuery(k).where('requestId', requested.requestId).limit(1))).at(0) + if (stored === undefined) return invalid() + const result = receipt(stored, true) + if ( + result.binding !== bound || + result.highWater !== requested.highWater || + result.expiresAt !== requested.expiresAt || + result.expiresAt <= time || + compareSnapshotJournalRevisions(result.highWater, state.floor) < 0 || + compareSnapshotJournalRevisions(result.floor, state.floor) > 0 + ) + return invalid() + return result +} + +/** At most 64 indexed expired receipts per caller-owned transaction. Retiring a + * receipt never reopens its expired request; capacity is released only on commit. */ +export async function collectSnapshotJournalReceipts(k: Knex): Promise { + await retention(k, true) + const time = await now(k) + const query = k('snapshot_journal_receipts') + .where('expiresAt', '<=', time) + .select(k.raw('substr(??,1,65) AS ??', ['requestId', 'requestId'])) + .orderBy(['expiresAt', 'requestId']) + .limit(64) + if (!local(k)) query.forUpdate().noWait() + const rows: Array<{ requestId: unknown }> = await query + if (rows.some(row => typeof row.requestId !== 'string' || !digestPattern.test(row.requestId))) return invalid() + if (rows.length) + await k('snapshot_journal_receipts') + .whereIn( + 'requestId', + rows.map(row => row.requestId as string) + ) + .delete() + return rows.length +} + +/** Advance only inside a fresh caller-owned transaction on an already validated, + * complete generation. The global writer reservation precedes retention and + * receipt locks, matching capture's order. Undefined disables publication: the + * caller must commit the clock invalidation and cannot collect that generation. + * Expired receipts continue to occupy capacity until their bounded collector + * commits, but cannot pin continuity after database-clock expiry. + */ +export async function advanceSnapshotJournalFloor( + k: Knex, + requested: SnapshotJournalRevision +): Promise< + | { floor: SnapshotJournalRevision; highWater: SnapshotJournalRevision; liveReceipts: number; examined: number } + | undefined +> { + const floor = snapshotJournalRevision(requested) + transaction(k) + const highWater = await reserveSnapshotJournalCaptureFence(k) + if (highWater === undefined) return undefined + const state = await retention(k, true), + time = await now(k) + if ( + compareSnapshotJournalRevisions(state.floor, highWater) > 0 || + compareSnapshotJournalRevisions(floor, state.floor) < 0 || + compareSnapshotJournalRevisions(floor, highWater) > 0 + ) + return invalid() + const query = receiptQuery(k) + .orderBy('requestId') + .limit(state.receiptLimit + 1) + if (!local(k)) query.forUpdate().noWait() + const rows: Array> = await query + if (rows.length > state.receiptLimit) return invalid() + let liveReceipts = 0 + for (const row of rows) { + const existing = receipt(row, true) + if (compareSnapshotJournalRevisions(existing.floor, state.floor) > 0) return invalid() + if (existing.expiresAt > time) { + if ( + compareSnapshotJournalRevisions(existing.highWater, state.floor) < 0 || + compareSnapshotJournalRevisions(existing.highWater, highWater) > 0 || + compareSnapshotJournalRevisions(floor, existing.highWater) > 0 + ) + return invalid() + liveReceipts++ + } + } + if ((await k('snapshot_journal_retention').where('id', 1).where('floor', state.floor).update({ floor })) !== 1) + return invalid() + return { floor, highWater, liveReceipts, examined: rows.length } +} + +/** Internal collector lock; callers reserve the global writer clock first. */ +export async function lockSnapshotJournalRetention(k: Knex): Promise { + return await retention(k, true) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceiptMysql.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceiptMysql.test.ts new file mode 100644 index 000000000..e4397576f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalReceiptMysql.test.ts @@ -0,0 +1,315 @@ +import { knex, type Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { + snapshotJournalReceiptDdl, + snapshotJournalReceiptBinding, + recordSnapshotJournalReceipt, + readSnapshotJournalReceipt, + collectSnapshotJournalReceipts, + type SnapshotJournalReceiptBinding +} from './SnapshotJournalReceipt' +import { snapshotJournalRevision } from './SnapshotJournalRevision' + +const binding: SnapshotJournalReceiptBinding = { + backend: '11'.repeat(32), + epoch: '12345678-1234-4234-9234-123456789012', + source: '22'.repeat(32), + schema: '33'.repeat(32), + storageIdentity: 'synthetic-storage', + identityKey: '02' + '44'.repeat(32), + userId: 1, + chain: 'test' +} +const time = 1700000000000 +const input = (id: number) => ({ + requestId: id.toString(16).padStart(64, '0'), + highWater: snapshotJournalRevision('9007199254740993'), + expiresAt: time + 60000 +}) +interface Query { + sql: string + bindings: Knex.RawBinding[] + method: string + response?: unknown +} + +function stringifyBigints(rows: Array>): void { + for (const row of rows) + for (const field of ['expiresAt', 'receiptLifetimeMs']) + if (row[field] !== undefined) row[field] = String(row[field]) +} + +function overrideBigints( + rows: Array>, + values: Partial> +): void { + for (const row of rows) + for (const field of ['expiresAt', 'receiptLifetimeMs'] as const) + if (row[field] !== undefined && Object.hasOwn(values, field)) row[field] = values[field] +} + +// Real MySQL query compilation/response processing, backed by SQLite for DML +// and rollback. The independent native fixture proves MySQL isolation/locks. +async function fixture(strings: boolean) { + const db = knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }), + k = knex({ client: 'mysql2' }) + for (const sql of snapshotJournalReceiptDdl(db)) await db.raw(sql) + await db('snapshot_journal_retention').insert({ id: 1, floor: '0', receiptLimit: 2, receiptLifetimeMs: 2592000000 }) + const queries: Query[] = [], + state = { + failLock: false, + now: time, + storedBigints: {} as Partial> + } + const connection = { + __knexUid: 'receipt-driver', + query: ( + options: { sql: string }, + bindings: Knex.RawBinding[] | undefined, + callback: (error: unknown, rows?: unknown, fields?: unknown) => void + ) => { + void k.client._query(connection, { sql: options.sql, bindings: bindings ?? [], method: 'raw' }).then( + (result: { response: [unknown, unknown] }) => callback(null, ...result.response), + (error: unknown) => callback(error) + ) + } + } + k.client.acquireConnection = async () => connection + k.client.releaseConnection = async () => undefined + k.client._query = async (_connection: unknown, q: Query) => { + queries.push({ sql: q.sql, bindings: q.bindings, method: q.method }) + const respond = (rows: unknown) => { + q.response = [rows, []] + return q + } + if (state.failLock && /(?:for update|FOR SHARE) NOWAIT$/i.test(q.sql)) + throw Object.assign(new Error('owned fixture lock busy'), { code: 'ER_LOCK_NOWAIT' }) + if (q.sql === 'SELECT FLOOR(UNIX_TIMESTAMP(CURRENT_TIMESTAMP(3)) * 1000) AS now') + return respond([{ now: strings ? String(state.now) : state.now }]) + const sql = q.sql.replace(/ (?:for (?:share|update)(?: nowait)?|lock in share mode)$/i, '') + const result = await db.raw(sql, q.bindings) + if (Array.isArray(result)) { + if (strings) stringifyBigints(result) + overrideBigints(result, state.storedBigints) + return respond(result) + } + return respond({ affectedRows: result?.changes ?? 0, insertId: result?.lastInsertRowid ?? 0 }) + } + return { + db, + k, + queries, + state, + close: async () => { + await k.destroy() + await db.destroy() + } + } +} + +test.each([false, true])( + 'MySQL receipt operations preserve exact values and retry/rollback semantics (string BIGINT: %s)', + async strings => { + const f = await fixture(strings) + try { + const a = input(1), + b = input(2), + expected = { ...a, binding: snapshotJournalReceiptBinding(binding), floor: '0' } + expect(await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a))).toEqual(expected) + expect(await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a))).toEqual(expected) + expect(await f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).toEqual(expected) + await expect( + f.k.transaction(async t => { + await recordSnapshotJournalReceipt(t, binding, b) + throw Error('rollback') + }) + ).rejects.toThrow('rollback') + expect(await f.db('snapshot_journal_receipts')).toHaveLength(1) + await expect(f.k.transaction(t => readSnapshotJournalReceipt(t, { ...binding, userId: 2 }, a))).rejects.toThrow() + await f.db('snapshot_journal_retention').update({ floor: '9007199254740994' }) + await expect(f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).rejects.toThrow() + expect(f.queries.some(q => /snapshot_journal_retention` limit \? for update nowait$/i.test(q.sql))).toBe(true) + const receiptReads = f.queries.filter( + q => q.sql.startsWith('select ') && q.sql.includes('snapshot_journal_receipts') + ) + expect(receiptReads.length).toBeGreaterThan(0) + expect(receiptReads.every(q => /FOR (?:SHARE|UPDATE) NOWAIT$/i.test(q.sql))).toBe(true) + expect(receiptReads.every(q => !q.sql.includes('select *'))).toBe(true) + } finally { + await f.close() + } + } +) + +test('MySQL expired capacity remains charged until collection commits', async () => { + const f = await fixture(true) + try { + const a = input(1), + b = input(2), + c = input(3) + await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, b)) + await f.db('snapshot_journal_receipts').where('requestId', a.requestId).update({ expiresAt: 1 }) + await expect(f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, c))).rejects.toThrow() + await expect( + f.k.transaction(async t => { + expect(await collectSnapshotJournalReceipts(t)).toBe(1) + throw Error('collection rollback') + }) + ).rejects.toThrow('collection rollback') + await expect(f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, c))).rejects.toThrow() + expect(await f.k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(1) + await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, c)) + expect(await f.k.transaction(t => readSnapshotJournalReceipt(t, binding, b))).toMatchObject(b) + const capacityQueries = f.queries.filter(q => q.sql.startsWith('select 1 as `occupied`')) + expect(capacityQueries.length).toBeGreaterThan(0) + expect(capacityQueries.every(q => /order by `requestId` asc limit \? for update nowait$/i.test(q.sql))).toBe(true) + expect(capacityQueries.every(q => q.bindings.at(-1) === 3)).toBe(true) + expect( + f.queries.some( + q => + q.sql.includes('order by `expiresAt` asc, `requestId` asc limit ? for update nowait') && + q.bindings.at(-1) === 64 + ) + ).toBe(true) + } finally { + await f.close() + } +}) + +test('MySQL lock refusal propagates and writes no partial receipt', async () => { + const f = await fixture(false) + try { + f.state.failLock = true + await expect(f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, input(1)))).rejects.toMatchObject({ + code: 'ER_LOCK_NOWAIT' + }) + await expect(f.k.transaction(t => readSnapshotJournalReceipt(t, binding, input(1)))).rejects.toMatchObject({ + code: 'ER_LOCK_NOWAIT' + }) + await expect(f.k.transaction(t => collectSnapshotJournalReceipts(t))).rejects.toMatchObject({ + code: 'ER_LOCK_NOWAIT' + }) + expect(await f.db('snapshot_journal_receipts')).toEqual([]) + } finally { + await f.close() + } +}) + +test('bounded stored projections reject oversized identities and revisions', async () => { + const f = await fixture(true) + try { + const a = input(1) + await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + const stored = await f.db('snapshot_journal_receipts').first() + for (const change of [ + { binding: 'a'.repeat(100000) }, + { highWater: '1'.repeat(100000) }, + { floor: '1'.repeat(100000) } + ]) { + await f.db('snapshot_journal_receipts').update(change) + await expect(f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).rejects.toThrow() + await f.db('snapshot_journal_receipts').update(stored) + } + await f.db('snapshot_journal_receipts').update({ requestId: 'f'.repeat(100000), expiresAt: 1 }) + await expect(f.k.transaction(t => collectSnapshotJournalReceipts(t))).rejects.toThrow() + expect(await f.db('snapshot_journal_receipts')).toHaveLength(1) + } finally { + await f.close() + } +}) + +test('MySQL receipt tables carry the expiry index in the atomic table definition', async () => { + const k = knex({ client: 'mysql2' }) + try { + const ddl = snapshotJournalReceiptDdl(k) + expect(ddl).toHaveLength(2) + expect(ddl[1]).toContain('KEY snapshot_journal_receipts_expiry(expiresAt,requestId)') + expect( + ddl.every(sql => sql.endsWith('ENGINE=InnoDB DEFAULT CHARACTER SET ascii COLLATE ascii_bin ROW_FORMAT=DYNAMIC')) + ).toBe(true) + } finally { + await k.destroy() + } +}) + +test('MySQL legacy alias retains current-read receipt and retention semantics', async () => { + const f = await fixture(true) + try { + f.k.client.config.client = 'mysql' + const a = input(1) + expect(snapshotJournalReceiptDdl(f.k)).toHaveLength(2) + await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + expect(await f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).toMatchObject(a) + f.state.now = a.expiresAt + expect(await f.k.transaction(t => collectSnapshotJournalReceipts(t))).toBe(1) + expect(await f.db('snapshot_journal_receipts')).toEqual([]) + } finally { + await f.close() + } +}) + +test('MySQL driver BIGINT responses reject noncanonical strings and nonprimitive numeric lookalikes', async () => { + const f = await fixture(true) + try { + const a = input(1) + await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + for (const field of ['expiresAt', 'receiptLifetimeMs'] as const) { + const value = field === 'expiresAt' ? a.expiresAt : 2592000000 + const invalid: unknown[] = [ + '0' + value, + '+' + value, + ' ' + value, + value + ' ', + value / 1000 + 'e3', + value + '.0', + 0, + -1, + 1.5, + Number.MAX_SAFE_INTEGER + 1, + Number.NaN, + Number.POSITIVE_INFINITY, + { valueOf: () => value, toString: () => String(value) } + ] + for (const response of invalid) { + f.state.storedBigints = { [field]: response } + await expect(f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + await expect(f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + } + f.state.storedBigints = { [field]: String(value) } + expect(await f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).toMatchObject(a) + f.state.storedBigints = {} + } + expect(await f.db('snapshot_journal_receipts')).toHaveLength(1) + } finally { + await f.close() + } +}) + +test('MySQL receipt reads acquire shared nonwaiting locks and preserve the recorded expiry', async () => { + const f = await fixture(false) + try { + const a = input(1) + await f.k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + f.queries.length = 0 + expect(await f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).toMatchObject(a) + const retentionReads = f.queries.filter(q => q.sql.includes('from `snapshot_journal_retention`')) + expect(retentionReads).toHaveLength(1) + expect(retentionReads[0].sql.endsWith('FOR SHARE NOWAIT')).toBe(true) + await f.db('snapshot_journal_receipts').update({ expiresAt: a.expiresAt + 1 }) + await expect(f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + await f.db('snapshot_journal_receipts').update({ expiresAt: a.expiresAt }) + f.state.now = a.expiresAt + await expect(f.k.transaction(t => readSnapshotJournalReceipt(t, binding, a))).rejects.toThrow( + WERR_INVALID_OPERATION + ) + } finally { + await f.close() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevision.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevision.ts new file mode 100644 index 000000000..3a3f93512 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevision.ts @@ -0,0 +1,29 @@ +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' + +/** Common nonnegative INTEGER range for SQLite and MySQL journal positions. */ +export const MAX_SNAPSHOT_JOURNAL_REVISION = '9223372036854775807' + +declare const journalRevision: unique symbol +export type SnapshotJournalRevision = string & { readonly [journalRevision]: true } + +/** Canonical decimal strings preserve exact persisted and transported positions. */ +export function snapshotJournalRevision(value: unknown): SnapshotJournalRevision { + if ( + typeof value !== 'string' || + !/^(?:0|[1-9]\d{0,18})$/.test(value) || + (value.length === 19 && value > MAX_SNAPSHOT_JOURNAL_REVISION) + ) { + throw new WERR_INVALID_OPERATION('Invalid snapshot journal revision') + } + return value as SnapshotJournalRevision +} + +/** The caller must validate both values at each persisted or transport boundary. */ +export function compareSnapshotJournalRevisions( + left: SnapshotJournalRevision, + right: SnapshotJournalRevision +): -1 | 0 | 1 { + if (left === right) return 0 + if (left.length !== right.length) return left.length < right.length ? -1 : 1 + return left < right ? -1 : 1 +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevisionSql.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevisionSql.test.ts new file mode 100644 index 000000000..78e9e7d74 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevisionSql.test.ts @@ -0,0 +1,88 @@ +import { knex, type Knex } from 'knex' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' +import { snapshotJournalRevisionText, snapshotJournalRevisionOperand } from './SnapshotJournalRevisionSql' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' + +test('the SQL helpers preserve exact SQLite range and generation values', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + try { + await k.raw('CREATE TABLE revisions(revision INTEGER PRIMARY KEY,generation INTEGER NOT NULL)') + const values = ['9', '10', '9007199254740992', '9007199254740993', '9223372036854775807'] + for (const revision of values) await k('revisions').insert({ revision, generation: revision }) + const query = k('revisions') + .select({ + revisionText: snapshotJournalRevisionText(k, 'revisions.revision'), + generationText: snapshotJournalRevisionText(k, 'revisions.generation') + }) + .where('revision', '>', snapshotJournalRevisionOperand(k, snapshotJournalRevision('9007199254740992'))) + .orderBy('revision') + .limit(2) + expect(await query).toEqual(values.slice(3).map(value => ({ revisionText: value, generationText: value }))) + const sql = query.toSQL(), + plan: Array<{ detail: string }> = await k.raw('EXPLAIN QUERY PLAN ' + sql.sql, sql.bindings) + expect(plan.some(row => row.detail.includes('SEARCH revisions USING INTEGER PRIMARY KEY'))).toBe(true) + expect(plan.some(row => row.detail.includes('TEMP B-TREE'))).toBe(false) + expect(() => snapshotJournalRevisionOperand(k, '9223372036854775808' as SnapshotJournalRevision)).toThrow( + WERR_INVALID_OPERATION + ) + expect(() => snapshotJournalRevisionOperand(k, '1 OR 1=1' as SnapshotJournalRevision)).toThrow( + WERR_INVALID_OPERATION + ) + } finally { + await k.destroy() + } +}) + +test('MySQL operands and selected text use integer-preserving types and parameter bindings', async () => { + const k = knex({ client: 'mysql2' }) + try { + const query = k('revisions') + .select({ exactRevision: snapshotJournalRevisionText(k, 'revision') }) + .where('revision', '>', snapshotJournalRevisionOperand(k, snapshotJournalRevision('9007199254740992'))) + .orderBy('revision') + .limit(2) + .toSQL() + expect(query.sql).toContain('CAST(`revision` AS CHAR)') + expect(query.sql).toContain('CAST(? AS SIGNED)') + expect(query.bindings).toEqual(['9007199254740992', 2]) + } finally { + await k.destroy() + } +}) + +test.each(['pg', 'mssql', 'mysql-custom', undefined])('unknown driver %p refuses before issuing SQL', client => { + const k = { client: { config: { client } } } as unknown as Knex + expect(() => snapshotJournalRevisionText(k, 'revision')).toThrow('Unsupported snapshot journal SQL driver') + expect(() => snapshotJournalRevisionOperand(k, snapshotJournalRevision('0'))).toThrow( + 'Unsupported snapshot journal SQL driver' + ) +}) + +test.each([ + ['better-sqlite3', 'TEXT', 'INTEGER'], + ['sqlite3', 'TEXT', 'INTEGER'], + ['mysql', 'CHAR', 'SIGNED'], + ['mysql2', 'CHAR', 'SIGNED'] +])('driver alias %s preserves the explicit selected and operand types', async (client, selected, operand) => { + const k = knex({ client: 'mysql2' }) + // Compile through an installed client; alias selection must not open a connection. + k.client.config.client = client + const query = jest.fn() + k.on('query', query) + try { + const text = snapshotJournalRevisionText(k, 'revision').toSQL() + const value = snapshotJournalRevisionOperand(k, snapshotJournalRevision('9223372036854775807')).toSQL() + expect(text.sql).toBe('CAST(`revision` AS ' + selected + ')') + expect(text.bindings).toEqual([]) + expect(value.sql).toBe('CAST(? AS ' + operand + ')') + expect(value.bindings).toEqual(['9223372036854775807']) + expect(query).not.toHaveBeenCalled() + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevisionSql.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevisionSql.ts new file mode 100644 index 000000000..40aacd89f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalRevisionSql.ts @@ -0,0 +1,21 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' + +function sqlite(k: Knex): boolean { + const client = k.client.config.client + if (client === 'better-sqlite3' || client === 'sqlite3') return true + if (client === 'mysql' || client === 'mysql2') return false + throw new WERR_INVALID_OPERATION('Unsupported snapshot journal SQL driver') +} + +/** Select with an alias different from the integer column used by ORDER BY. */ +export function snapshotJournalRevisionText(k: Knex, column: string): Knex.Raw { + return k.raw(sqlite(k) ? 'CAST(?? AS TEXT)' : 'CAST(?? AS CHAR)', [column]) +} + +/** Explicit integer operands avoid implicit floating-point comparisons. */ +export function snapshotJournalRevisionOperand(k: Knex, value: SnapshotJournalRevision): Knex.Raw { + const revision = snapshotJournalRevision(value) + return k.raw(sqlite(k) ? 'CAST(? AS INTEGER)' : 'CAST(? AS SIGNED)', [revision]) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteClock.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteClock.test.ts new file mode 100644 index 000000000..2c99ec55f --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteClock.test.ts @@ -0,0 +1,245 @@ +import { knex, type Knex } from 'knex' +import { mkdtemp, rm } from 'node:fs/promises' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { + MAX_SNAPSHOT_JOURNAL_REVISION, + snapshotJournalRevision as rev, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' +import { snapshotJournalRevisionText } from './SnapshotJournalRevisionSql' +import { + installSnapshotJournalSqliteClock, + SNAPSHOT_JOURNAL_SQLITE_ADVANCE, + SNAPSHOT_JOURNAL_SQLITE_WRITABLE, + SNAPSHOT_JOURNAL_SQLITE_REVISION +} from './SnapshotJournalSqliteClock' + +function open(filename = ':memory:'): Knex { + return knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) +} + +async function install(k: Knex, ceiling: string): Promise { + await installSnapshotJournalSqliteClock(k, rev(ceiling)) + await k.raw('CREATE TABLE ordinary_rows(id INTEGER PRIMARY KEY,payload TEXT)') + await k.raw('CREATE TABLE journal_rows(id INTEGER PRIMARY KEY,revision INTEGER)') + await k.raw(`CREATE TRIGGER observe_ordinary_row AFTER INSERT ON ordinary_rows BEGIN + ${SNAPSHOT_JOURNAL_SQLITE_ADVANCE} + INSERT INTO journal_rows(id,revision) SELECT NEW.id,${SNAPSHOT_JOURNAL_SQLITE_REVISION} WHERE ${SNAPSHOT_JOURNAL_SQLITE_WRITABLE}; + END`) +} + +test('event-window exhaustion invalidates continuity while all ordinary rows commit', async () => { + const k = open() + try { + await install(k, '2') + for (const id of [1, 2, 3, 4]) await k('ordinary_rows').insert({ id, payload: 'preserve-' + id }) + expect(await k('ordinary_rows').orderBy('id')).toEqual([1, 2, 3, 4].map(id => ({ id, payload: 'preserve-' + id }))) + expect(await k('journal_rows').orderBy('id')).toEqual([ + { id: 1, revision: 1 }, + { id: 2, revision: 2 } + ]) + expect(await k('snapshot_journal_clock').first()).toEqual({ + id: 1, + revision: 2, + ceiling: 2, + enabled: 0, + reason: 'capacity-exhausted' + }) + } finally { + await k.destroy() + } +}) + +test('revision exhaustion never converts the clock to floating point or rejects the ordinary write', async () => { + const k = open() + try { + await install(k, MAX_SNAPSHOT_JOURNAL_REVISION) + await k('snapshot_journal_clock').update({ + revision: String(BigInt(MAX_SNAPSHOT_JOURNAL_REVISION) - 1n) + }) + await k('ordinary_rows').insert({ id: 1, payload: 'last representable journal event' }) + expect(await k('journal_rows').select({ revision: snapshotJournalRevisionText(k, 'revision') })).toEqual([ + { revision: MAX_SNAPSHOT_JOURNAL_REVISION } + ]) + await k('ordinary_rows').insert({ id: 2, payload: 'preserved after journal exhaustion' }) + expect(await k('ordinary_rows').orderBy('id').pluck('id')).toEqual([1, 2]) + expect(await k('snapshot_journal_clock').select('enabled', 'reason').first()).toEqual({ + enabled: 0, + reason: 'revision-exhausted' + }) + expect( + await k.raw('SELECT typeof(revision) kind,CAST(revision AS TEXT) revision FROM snapshot_journal_clock') + ).toEqual([{ kind: 'integer', revision: MAX_SNAPSHOT_JOURNAL_REVISION }]) + } finally { + await k.destroy() + } +}) + +test('rollback restores the source, journal and continuity flag together', async () => { + const k = open() + try { + await install(k, '1') + await k('ordinary_rows').insert({ id: 1, payload: 'committed' }) + const failure = new Error('rollback source transaction') + await expect( + k.transaction(async transaction => { + await transaction('ordinary_rows').insert({ id: 2, payload: 'rolled back' }) + expect(await transaction('snapshot_journal_clock').first('enabled')).toEqual({ enabled: 0 }) + throw failure + }) + ).rejects.toBe(failure) + expect(await k('snapshot_journal_clock').first()).toEqual({ + id: 1, + revision: 1, + ceiling: 1, + enabled: 1, + reason: null + }) + expect(await k('ordinary_rows')).toEqual([{ id: 1, payload: 'committed' }]) + expect(await k('journal_rows')).toEqual([{ id: 1, revision: 1 }]) + } finally { + await k.destroy() + } +}) + +test('an already pinned WAL view remains coherent across continuity invalidation', async () => { + const directory = await mkdtemp(join(tmpdir(), 'ts569-journal-clock-')) + const k = open(join(directory, 'clock.sqlite')), + writer = open(join(directory, 'clock.sqlite')) + let view: Knex.Transaction | undefined + try { + await k.raw('PRAGMA journal_mode=WAL') + await install(k, '1') + await k('ordinary_rows').insert({ id: 1, payload: 'before pin' }) + view = await k.transaction() + expect(await view('snapshot_journal_clock').first('enabled')).toEqual({ enabled: 1 }) + await writer('ordinary_rows').insert({ id: 2, payload: 'after pin' }) + expect(await writer('snapshot_journal_clock').first('enabled')).toEqual({ enabled: 0 }) + expect(await view('snapshot_journal_clock').first('enabled')).toEqual({ enabled: 1 }) + expect(await view('ordinary_rows')).toEqual([{ id: 1, payload: 'before pin' }]) + expect(await view('journal_rows')).toEqual([{ id: 1, revision: 1 }]) + } finally { + await view?.rollback() + await k.destroy() + await writer.destroy() + await rm(directory, { recursive: true, force: true }) + } +}) + +test.each(['0', '01', '-1', '9223372036854775808'])( + 'invalid event ceiling %s refuses before installation', + async ceiling => { + const k = open() + try { + await expect(installSnapshotJournalSqliteClock(k, ceiling as SnapshotJournalRevision)).rejects.toThrow() + expect(await k.schema.hasTable('snapshot_journal_clock')).toBe(false) + } finally { + await k.destroy() + } + } +) + +test('new installation refuses to adopt or reset an existing clock', async () => { + const k = open() + try { + await install(k, '1') + await k('ordinary_rows').insert({ id: 1 }) + await expect(installSnapshotJournalSqliteClock(k, rev('10'))).rejects.toThrow() + expect(await k('snapshot_journal_clock').first()).toEqual({ + id: 1, + revision: 1, + ceiling: 1, + enabled: 1, + reason: null + }) + } finally { + await k.destroy() + } +}) + +test('the SQLite clock refuses a MySQL connection before issuing SQL', async () => { + const k = knex({ client: 'mysql2' }) + const seen = jest.fn() + k.on('query', seen) + try { + await expect(installSnapshotJournalSqliteClock(k, rev('1'))).rejects.toThrow( + 'Snapshot journal clock requires SQLite' + ) + expect(seen).not.toHaveBeenCalled() + } finally { + await k.destroy() + } +}) + +test('native clock constraints refuse invalid continuity states atomically', async () => { + const k = open() + try { + await install(k, '1') + for (const change of [ + { enabled: 0, reason: null }, + { enabled: 1, reason: 'capacity-exhausted' }, + { enabled: 0, reason: 'unknown' }, + { revision: 2 }, + { revision: -1 }, + { revision: 0.5 } + ]) { + await expect(k('snapshot_journal_clock').update(change)).rejects.toThrow() + expect(await k('snapshot_journal_clock').first()).toEqual({ + id: 1, + revision: 0, + ceiling: 1, + enabled: 1, + reason: null + }) + } + } finally { + await k.destroy() + } +}) + +test.each(['sqlite3', 'better-sqlite3'])( + 'SQLite clock alias %s creates and advances an exact event window', + async client => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + k.client.config.client = client + try { + await installSnapshotJournalSqliteClock(k, rev('9007199254740993')) + expect( + await k('snapshot_journal_clock').select( + 'revision', + k.raw('CAST(ceiling AS TEXT) ceiling'), + 'enabled', + 'reason' + ) + ).toEqual([{ revision: 0, ceiling: '9007199254740993', enabled: 1, reason: null }]) + } finally { + await k.destroy() + } + } +) + +test('empty SQLite clock event window refuses with the stable error before DDL', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }), + query = jest.fn() + k.on('query', query) + try { + await expect(installSnapshotJournalSqliteClock(k, rev('0'))).rejects.toThrow('Empty snapshot journal event window') + expect(query).not.toHaveBeenCalled() + } finally { + await k.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteClock.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteClock.ts new file mode 100644 index 000000000..5639742dc --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteClock.ts @@ -0,0 +1,46 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { + MAX_SNAPSHOT_JOURNAL_REVISION, + snapshotJournalRevision, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' + +export const SNAPSHOT_JOURNAL_SQLITE_CLOCK = 'snapshot_journal_clock' + +export const SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL = `CREATE TABLE snapshot_journal_clock ( +id INTEGER NOT NULL PRIMARY KEY CHECK(id=1), +revision INTEGER NOT NULL CHECK(typeof(revision)='integer' AND revision>=0), +ceiling INTEGER NOT NULL CHECK(typeof(ceiling)='integer' AND ceiling>=revision), +enabled INTEGER NOT NULL CHECK(enabled IN (0,1)), +reason TEXT CHECK((enabled=1 AND reason IS NULL) OR (enabled=0 AND reason IS NOT NULL AND reason IN ('capacity-exhausted','revision-exhausted','key-out-of-range'))) +)` + +/** Advance only within the reserved event window; the source write may still commit. */ +export const SNAPSHOT_JOURNAL_SQLITE_ADVANCE = `UPDATE snapshot_journal_clock SET +revision=CASE WHEN revision=ceiling THEN 0 ELSE enabled END, +reason=CASE WHEN revision>=ceiling THEN CASE WHEN revision=${MAX_SNAPSHOT_JOURNAL_REVISION} THEN 'revision-exhausted' ELSE 'capacity-exhausted' END ELSE reason END +WHERE id=1 AND enabled=1; ` + +/** Guard every metadata write after advancing: the current event may invalidate the epoch. */ +export const SNAPSHOT_JOURNAL_SQLITE_WRITABLE = '(SELECT enabled FROM snapshot_journal_clock WHERE id=1)=1' +export const SNAPSHOT_JOURNAL_SQLITE_REVISION = '(SELECT revision FROM snapshot_journal_clock WHERE id=1)' + +/** New owned installation only. Recovery and epoch replacement belong to the migration. */ +export async function installSnapshotJournalSqliteClock(k: Knex, ceiling: SnapshotJournalRevision): Promise { + if (k.client.config.client !== 'sqlite3' && k.client.config.client !== 'better-sqlite3') { + throw new WERR_INVALID_OPERATION('Snapshot journal clock requires SQLite') + } + if (snapshotJournalRevision(ceiling) === '0') throw new WERR_INVALID_OPERATION('Empty snapshot journal event window') + await k.transaction(async transaction => { + await transaction.raw(SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL) + await transaction(SNAPSHOT_JOURNAL_SQLITE_CLOCK).insert({ + id: 1, + revision: 0, + ceiling, + enabled: 1, + reason: null + }) + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.test.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.test.ts new file mode 100644 index 000000000..a8a307521 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.test.ts @@ -0,0 +1,484 @@ +import * as observers from './SnapshotJournalSqliteObservers' +import { knex, type Knex } from 'knex' +import { StorageKnex } from '../../StorageKnex' +import { StorageProvider } from '../../StorageProvider' +import { seedArchiveClosure } from '../../../../test/utils/snapshotArchiveFixtures' +import { + installSnapshotJournalSqliteGeneration, + readSnapshotJournalSqliteGeneration, + completeSnapshotJournalSqliteGeneration, + SNAPSHOT_JOURNAL_SQLITE_GENERATION as metadata +} from './SnapshotJournalSqliteGeneration' +import { copySnapshotJournalBootstrapPage } from './SnapshotJournalBootstrap' +import { snapshotJournalRevision, type SnapshotJournalRevision } from './SnapshotJournalRevision' + +const journalReceiptPolicy = { receiptLimit: 128, receiptLifetimeMs: 2592000000 } + +let ceiling: SnapshotJournalRevision +beforeEach(() => { + ceiling = snapshotJournalRevision('1000000') +}) +async function fixture() { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + try { + await source.migrate('journal lifecycle', 'synthetic-journal-lifecycle') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)), + { user: foreign } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, foreign.userId) + return { k, source } + } catch (error) { + await source.destroy() + throw error + } +} +async function finish(k: Knex) { + for (let page = 0; page < 80; page++) if ((await copySnapshotJournalBootstrapPage(k, 1000000)).complete) return + throw new Error('bootstrap did not finish') +} + +test('atomic installation resumes its epoch and publishes only completed durable bootstrap', async () => { + const { k, source } = await fixture() + try { + const rows = await k('transactions'), + installed = await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + expect(installed).toMatchObject({ complete: false, enabled: true, ceiling }) + expect(installed.epoch).toMatch(/^[0-9a-f-]{36}$/) + await expect(completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toThrow('Invalid or unowned') + await copySnapshotJournalBootstrapPage(k, 1000000) + const progress = await k('snapshot_journal_bootstrap').first() + expect(await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).toEqual(installed) + expect(await k('snapshot_journal_bootstrap').first()).toEqual(progress) + await finish(k) + expect(await readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual(installed) + expect(await completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual({ + ...installed, + complete: true + }) + expect(await readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual({ ...installed, complete: true }) + expect(await k('transactions')).toEqual(rows) + } finally { + await source.destroy() + } +}) +test.each([ + 'CREATE TABLE snapshot_journal_clock(id INTEGER)', + 'CREATE VIEW snapshot_journal_scope_0_INSERT AS SELECT 1', + 'CREATE INDEX snapshot_journal_foreign ON users(identityKey)', + "CREATE TRIGGER snapshot_journal_foreign AFTER INSERT ON users BEGIN SELECT 'foreign'; END" +])('first installation refuses unrelated reserved objects: %s', async ddl => { + const { k, source } = await fixture() + try { + await k.raw(ddl) + const before = await k('sqlite_master').orderBy(['type', 'name']) + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(await k('sqlite_master').orderBy(['type', 'name'])).toEqual(before) + } finally { + await source.destroy() + } +}) +test.each(['observer', 'index', 'same-name view', 'unknown reserved'])( + 'resume refuses %s ownership drift without source mutation', + async kind => { + const { k, source } = await fixture() + try { + await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + if (kind === 'observer') await k.raw('DROP TRIGGER snapshot_journal_scope_0_INSERT') + if (kind === 'index') await k.raw('DROP INDEX snapshot_journal_scope_page') + if (kind === 'same-name view') await k.raw('CREATE VIEW snapshot_journal_scope_0_INSERT AS SELECT 1') + if (kind === 'unknown reserved') await k.raw('CREATE TABLE snapshot_journal_foreign(id INTEGER)') + const before = await k('sqlite_master').orderBy(['type', 'name']), + rows = await k('transactions') + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(await k('sqlite_master').orderBy(['type', 'name'])).toEqual(before) + expect(await k('transactions')).toEqual(rows) + } finally { + await source.destroy() + } + } +) +test('complete source binding includes user observers and preserves literal whitespace', async () => { + const { k, source } = await fixture() + try { + await k.raw("CREATE TRIGGER application_user AFTER INSERT ON users BEGIN SELECT 'a b'; END") + await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + await k.raw('DROP TRIGGER application_user') + await k.raw("CREATE TRIGGER application_user AFTER INSERT ON users BEGIN SELECT 'a b'; END") + await expect(readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toThrow('Invalid or unowned') + } finally { + await source.destroy() + } +}) +test.each([ + 'metadata missing', + 'metadata epoch', + 'metadata source', + 'metadata ceiling', + 'clock missing', + 'clock ceiling', + 'bootstrap missing', + 'bootstrap extra', + 'bootstrap cursor', + 'false completion' +])('persisted %s damage refuses', async kind => { + const { k, source } = await fixture() + try { + await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + if (kind === 'metadata missing') await k(metadata).delete() + if (kind === 'metadata epoch') await k(metadata).update({ epoch: 'foreign' }) + if (kind === 'metadata source') await k(metadata).update({ source: '0'.repeat(64) }) + if (kind === 'metadata ceiling') await k(metadata).update({ ceiling: '01' }) + if (kind === 'clock missing') await k('snapshot_journal_clock').delete() + if (kind === 'clock ceiling') await k('snapshot_journal_clock').update({ ceiling: 999 }) + if (kind === 'bootstrap missing') await k('snapshot_journal_bootstrap').delete() + if (kind === 'bootstrap extra') await k('snapshot_journal_bootstrap').update({ stream: 0.5 }) + if (kind === 'bootstrap cursor') await k('snapshot_journal_bootstrap').update({ cursor: 'x'.repeat(2049) }) + if (kind === 'false completion') await k(metadata).update({ complete: 1 }) + await expect(readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toThrow() + } finally { + await source.destroy() + } +}) +test('configured event exhaustion preserves source writes and refuses completion', async () => { + const { k, source } = await fixture() + try { + const installed = await installSnapshotJournalSqliteGeneration( + k, + snapshotJournalRevision('1'), + journalReceiptPolicy + ) + await k('tx_labels').where('txLabelId', 1).update({ label: 'first' }) + await k('tx_labels').where('txLabelId', 1).update({ label: 'ordinary after exhaustion' }) + expect((await k('tx_labels').where('txLabelId', 1).first()).label).toBe('ordinary after exhaustion') + expect(await readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual({ ...installed, enabled: false }) + await k('snapshot_journal_bootstrap').update({ stream: 17, cursor: null, rowLimit: 1000000 }) + await expect(completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toThrow('Invalid or unowned') + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await source.destroy() + } +}) +test.each([ + 'CREATE TABLE snapshot_journal_scope', + 'CREATE TRIGGER snapshot_journal_physical_12_DELETE', + 'insert into `snapshot_journal_generation`' +])('failure during %s rolls back every installation object', async point => { + const { k, source } = await fixture() + try { + const before = await k('sqlite_master').orderBy(['type', 'name']), + failure = new Error('synthetic installation failure') + const listener = (query: { sql: string }) => { + if (query.sql.startsWith(point)) throw failure + } + k.on('query', listener) + try { + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toBe(failure) + } finally { + k.off('query', listener) + } + expect(await k('sqlite_master').orderBy(['type', 'name'])).toEqual(before) + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).resolves.toMatchObject({ + complete: false, + enabled: true + }) + } finally { + await source.destroy() + } +}) +test('completion rollback retains the unfinished generation and can resume', async () => { + const { k, source } = await fixture() + try { + const installed = await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + await finish(k) + const failure = new Error('synthetic completion failure'), + listener = (query: { sql: string }) => { + if (query.sql.startsWith('update `snapshot_journal_generation`')) throw failure + } + k.on('query', listener) + try { + await expect(completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toBe(failure) + } finally { + k.off('query', listener) + } + expect(await readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual(installed) + await expect(completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).resolves.toMatchObject({ + complete: true + }) + } finally { + await source.destroy() + } +}) +test('unsupported client and missing published SQLite prerequisites refuse before installation', async () => { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true + }) + try { + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(await k('sqlite_master')).toEqual([]) + k.client.config.client = 'mysql2' + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + } finally { + await k.destroy() + } +}) + +test.each(['zero ceiling', 'invalid clock reason', 'replaced observer'])( + 'owned generation refuses incompatible persisted %s', + async kind => { + const { k, source } = await fixture() + try { + await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + if (kind === 'zero ceiling') await k(metadata).update({ ceiling: '0' }) + if (kind === 'invalid clock reason') { + await k.raw('PRAGMA ignore_check_constraints=ON') + await k('snapshot_journal_clock').update({ reason: 'unexpected' }) + await k.raw('PRAGMA ignore_check_constraints=OFF') + } + if (kind === 'replaced observer') { + await k.raw('DROP TRIGGER snapshot_journal_scope_0_INSERT') + await k.raw('CREATE TRIGGER snapshot_journal_scope_0_INSERT AFTER INSERT ON transactions BEGIN SELECT 1; END') + } + const before = await k('transactions') + await expect(readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toThrow('Invalid or unowned') + expect(await k('transactions')).toEqual(before) + } finally { + await source.destroy() + } + } +) +test.each(['rows', 'definition', 'aggregate'])('source binding refuses an oversized native schema %s', async kind => { + const { k, source } = await fixture() + try { + if (kind === 'rows') { + for (let i = 0; i < 513; i++) await k.raw('CREATE INDEX ?? ON users(identityKey)', ['application_index_' + i]) + } else { + const count = kind === 'aggregate' ? 20 : 1 + const literal = 'x'.repeat(kind === 'aggregate' ? 60000 : 65536) + for (let i = 0; i < count; i++) + await k.raw( + 'CREATE TRIGGER application_observer_' + i + " AFTER UPDATE ON users BEGIN SELECT '" + literal + "'; END" + ) + } + const before = await k('sqlite_master').select('type', 'name', 'sql').orderBy(['type', 'name']) + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(await k('sqlite_master').select('type', 'name', 'sql').orderBy(['type', 'name'])).toEqual(before) + } finally { + await source.destroy() + } +}) + +test('invalid generated DDL cannot enter the persisted SQLite ownership plan', async () => { + const { k, source } = await fixture() + const spy = jest + .spyOn(observers, 'snapshotJournalSqliteObserverSql') + .mockResolvedValue(['CREATE TABLE application_table(id INTEGER)']) + try { + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(await k('sqlite_master').where('name', 'application_table')).toEqual([]) + } finally { + spy.mockRestore() + await source.destroy() + } +}) +test.each(['users', 'settings'])('source metadata must include %s in the ownership binding', async table => { + const { k, source } = await fixture() + let altered = false + const listener = (rows: Array<{ type: string; name: string }>, query: { sql: string }) => { + if (query.sql.includes('substr(sql,1,65537) AS sql') && query.sql.includes('where `tbl_name` in')) { + const index = rows.findIndex(row => row.type === 'table' && row.name === table) + expect(index).toBeGreaterThanOrEqual(0) + rows.splice(index, 1) + altered = true + } + } + k.on('query-response', listener) + try { + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned' + ) + expect(altered).toBe(true) + } finally { + k.off('query-response', listener) + await source.destroy() + } +}) + +test('legacy SQLite alias preserves installation identity through completion', async () => { + const { k, source } = await fixture() + k.client.config.client = 'sqlite3' + try { + const installed = await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + await finish(k) + expect(await completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual({ + ...installed, + complete: true + }) + expect(await readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual({ ...installed, complete: true }) + } finally { + await source.destroy() + } +}) + +test.each(['install', 'read', 'complete'])( + 'SQLite generation %s refuses a foreign client before I/O', + async operation => { + const transaction = jest.fn(), + raw = jest.fn() + const k = { client: { config: { client: 'mysql2' } }, transaction, raw } as unknown as Knex + const operations = { + install: () => installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy), + read: () => readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy), + complete: () => completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy) + } + const result = operations[operation as keyof typeof operations]() + await expect(result).rejects.toThrow('Invalid or unowned SQLite snapshot journal generation') + expect(transaction).not.toHaveBeenCalled() + expect(raw).not.toHaveBeenCalled() + } +) + +test('empty SQLite generation event window refuses before transaction admission', async () => { + const transaction = jest.fn(), + k = { client: { config: { client: 'sqlite3' } }, transaction } as unknown as Knex + await expect( + installSnapshotJournalSqliteGeneration(k, snapshotJournalRevision('0'), journalReceiptPolicy) + ).rejects.toThrow('Invalid or unowned SQLite snapshot journal generation') + expect(transaction).not.toHaveBeenCalled() +}) + +test.each(['prefix', 'suffix'])( + 'SQLite generation rejects an epoch with a valid UUID only as a %s substring', + async side => { + const { k, source } = await fixture() + try { + const installed = await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + await k(metadata).update({ + epoch: side === 'prefix' ? installed.epoch + 'x' : 'x' + installed.epoch + }) + await expect(readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned SQLite snapshot journal generation' + ) + } finally { + await source.destroy() + } + } +) + +test.each(['revision-exhausted', 'key-out-of-range'])( + 'SQLite %s state remains readable but cannot publish completion', + async reason => { + const { k, source } = await fixture() + try { + const installed = await installSnapshotJournalSqliteGeneration(k, ceiling, journalReceiptPolicy) + await k('snapshot_journal_clock').update({ enabled: 0, reason }) + expect(await readSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).toEqual({ + ...installed, + enabled: false + }) + await expect(completeSnapshotJournalSqliteGeneration(k, journalReceiptPolicy)).rejects.toThrow( + 'Invalid or unowned SQLite snapshot journal generation' + ) + } finally { + await source.destroy() + } + } +) + +test('receipt policy is explicit, detached and immutable throughout generation resume', async () => { + const { k, source } = await fixture() + try { + const policy = { receiptLimit: 2, receiptLifetimeMs: 1000 } + const opening = installSnapshotJournalSqliteGeneration(k, ceiling, policy) + policy.receiptLimit = 3 + const installed = await opening + const original = { receiptLimit: 2, receiptLifetimeMs: 1000 } + expect(await k('snapshot_journal_retention').first()).toEqual({ id: 1, floor: '0', ...original }) + expect(await installSnapshotJournalSqliteGeneration(k, ceiling, original)).toEqual(installed) + for (const changed of [policy, { ...original, receiptLifetimeMs: 1001 }]) { + await expect(installSnapshotJournalSqliteGeneration(k, ceiling, changed)).rejects.toThrow('Invalid or unowned') + await expect(readSnapshotJournalSqliteGeneration(k, changed)).rejects.toThrow('Invalid or unowned') + await expect(completeSnapshotJournalSqliteGeneration(k, changed)).rejects.toThrow('Invalid or unowned') + } + expect(await k('snapshot_journal_retention').first()).toEqual({ id: 1, floor: '0', ...original }) + } finally { + await source.destroy() + } +}) + +test.each([ + null, + undefined, + [], + {}, + { receiptLimit: 0, receiptLifetimeMs: 1 }, + { receiptLimit: 129, receiptLifetimeMs: 1 }, + { receiptLimit: 1.5, receiptLifetimeMs: 1 }, + { receiptLimit: 1, receiptLifetimeMs: 0 }, + { receiptLimit: 1, receiptLifetimeMs: 2592000001 }, + { receiptLimit: 1, receiptLifetimeMs: NaN } +])('invalid receipt installation policy %p fails before source access', async policy => { + const k = knex({ client: 'better-sqlite3', connection: { filename: ':memory:' }, useNullAsDefault: true }) + const queries: string[] = [] + k.on('query', (query: { sql: string }) => queries.push(query.sql)) + try { + await expect( + installSnapshotJournalSqliteGeneration(k, ceiling, policy as typeof journalReceiptPolicy) + ).rejects.toThrow() + await expect(readSnapshotJournalSqliteGeneration(k, policy as typeof journalReceiptPolicy)).rejects.toThrow() + await expect(completeSnapshotJournalSqliteGeneration(k, policy as typeof journalReceiptPolicy)).rejects.toThrow() + expect(queries).toEqual([]) + } finally { + await k.destroy() + } +}) + +test('generation validates the receipt floor and capacity with bounded metadata reads', async () => { + const { k, source } = await fixture(), + policy = { receiptLimit: 2, receiptLifetimeMs: 1000 } + try { + await installSnapshotJournalSqliteGeneration(k, ceiling, policy) + const baseline = await k('transactions') + for (const floor of ['01', '9223372036854775807']) { + await k('snapshot_journal_retention').update({ floor }) + await expect(readSnapshotJournalSqliteGeneration(k, policy)).rejects.toThrow() + } + await k('snapshot_journal_retention').update({ floor: '0' }) + await k('snapshot_journal_receipts').insert( + [1, 2, 3].map(id => ({ + requestId: id.toString(16).padStart(64, '0'), + binding: 'a'.repeat(64), + highWater: '1', + floor: '0', + expiresAt: 1 + })) + ) + await expect(readSnapshotJournalSqliteGeneration(k, policy)).rejects.toThrow('Invalid or unowned') + await k('snapshot_journal_receipts').where('requestId', '3'.padStart(64, '0')).delete() + expect((await readSnapshotJournalSqliteGeneration(k, policy)).enabled).toBe(true) + expect(await k('transactions')).toEqual(baseline) + } finally { + await source.destroy() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.ts new file mode 100644 index 000000000..32e1b6bd9 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.ts @@ -0,0 +1,279 @@ +import { + snapshotJournalReceiptDdl, + snapshotJournalReceiptPolicy, + type SnapshotJournalReceiptPolicy +} from './SnapshotJournalReceipt' +import type { Knex } from 'knex' +import { createHash, randomUUID } from 'node:crypto' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { numeric } from '../../schema/snapshotSqliteMembership' +import { names, metadata as indexMetadata } from '../../schema/snapshotSqliteIndexGeneration' +import { readGenerationIndexState } from '../../schema/snapshotSqliteIndexState' +import { SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL } from './SnapshotJournalSqliteClock' +import { + snapshotJournalSqliteObserverSql, + SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL +} from './SnapshotJournalSqliteObservers' +import { SNAPSHOT_JOURNAL_BOOTSTRAP_DDL, validSnapshotJournalBootstrapBudget } from './SnapshotJournalBootstrap' +import { + snapshotJournalRevision, + compareSnapshotJournalRevisions, + type SnapshotJournalRevision +} from './SnapshotJournalRevision' + +export const SNAPSHOT_JOURNAL_SQLITE_GENERATION = 'snapshot_journal_generation' +export const SNAPSHOT_JOURNAL_SQLITE_GENERATION_DDL = + 'CREATE TABLE snapshot_journal_generation(id INTEGER NOT NULL PRIMARY KEY CHECK(id=1),version INTEGER NOT NULL CHECK(version=1),epoch TEXT NOT NULL,source TEXT NOT NULL,ceiling TEXT NOT NULL,complete INTEGER NOT NULL CHECK(complete IN (0,1)))' +interface ObjectDefinition { + type: string + name: string + sql: string +} +interface SchemaObject { + type: string + name: string + tbl_name: string + sql: string | null +} +interface Plan { + source: string + objects: ObjectDefinition[] +} +export interface SnapshotJournalSqliteGeneration { + epoch: string + source: string + ceiling: SnapshotJournalRevision + complete: boolean + enabled: boolean +} +function invalid(): never { + throw new WERR_INVALID_OPERATION('Invalid or unowned SQLite snapshot journal generation') +} +const local = (k: Knex) => k.client.config.client === 'sqlite3' || k.client.config.client === 'better-sqlite3' +async function plan(k: Knex, config?: Knex.MigratorConfig): Promise { + if (!local(k) || (await readGenerationIndexState(k, config)) !== 'v2') return invalid() + const ddl = [ + SNAPSHOT_JOURNAL_SQLITE_GENERATION_DDL, + SNAPSHOT_JOURNAL_SQLITE_CLOCK_DDL, + ...SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL, + SNAPSHOT_JOURNAL_BOOTSTRAP_DDL, + ...snapshotJournalReceiptDdl(k), + ...(await snapshotJournalSqliteObserverSql(k)) + ] + const objects = ddl.map(sql => { + const match = /^CREATE (TABLE|INDEX|TRIGGER) (snapshot_journal_\w+)/.exec(sql) + if (!match) return invalid() + return { type: match[1].toLowerCase(), name: match[2], sql } + }) + const sources = [ + ...numeric.map(source => source.table), + 'tx_labels_map', + 'output_tags_map', + 'certificate_fields', + 'users', + 'settings', + ...Object.values(names) + ] + const rows: SchemaObject[] = await k('sqlite_master') + .whereIn('tbl_name', sources) + .select('type', 'name', 'tbl_name', k.raw('substr(sql,1,65537) AS sql')) + .orderBy(['type', 'name']) + .limit(513) + if (rows.length > 512 || rows.some(row => row.sql !== null && Buffer.byteLength(row.sql, 'utf8') > 65536)) + return invalid() + if (['users', 'settings'].some(table => !rows.some(row => row.type === 'table' && row.name === table))) + return invalid() + const expectedNames = new Set(objects.map(object => object.name)) + const source = JSON.stringify(rows.filter(row => !expectedNames.has(row.name))) + if (Buffer.byteLength(source, 'utf8') > 1048576) return invalid() + return { + source: createHash('sha256').update('snapshot-journal-sqlite-source-v1\n').update(source).digest('hex'), + objects + } +} +async function reserved(k: Knex): Promise { + return await k('sqlite_master') + .whereRaw('lower(substr(name,1,17))=?', ['snapshot_journal_']) + .select('type', 'name', 'tbl_name', k.raw('substr(sql,1,65537) AS sql')) + .orderBy(['type', 'name']) + .limit(513) +} +async function validateBootstrap(k: Knex, complete: number): Promise { + const positions = await k('snapshot_journal_bootstrap').select('*').limit(2) + if (positions.length !== 1) return invalid() + const position = positions[0] + if ( + position.id !== 1 || + !Number.isInteger(position.stream) || + position.stream < 0 || + position.stream > 17 || + !( + position.cursor === null || + (typeof position.cursor === 'string' && Buffer.byteLength(position.cursor, 'utf8') <= 2048) + ) || + (position.stream === 17 && position.cursor !== null) || + (complete === 1 && position.stream !== 17) || + !validSnapshotJournalBootstrapBudget(position) || + (position.rowLimit === null && (position.stream !== 0 || position.cursor !== null)) + ) + return invalid() +} +async function validateRetention( + k: Knex, + policy: SnapshotJournalReceiptPolicy, + ceiling: SnapshotJournalRevision +): Promise { + const retention = await k('snapshot_journal_retention') + .select('id', 'receiptLimit', 'receiptLifetimeMs', k.raw('substr(??,1,20) AS ??', ['floor', 'floor'])) + .limit(2) + if ( + retention.length !== 1 || + retention[0].id !== 1 || + retention[0].receiptLimit !== policy.receiptLimit || + retention[0].receiptLifetimeMs !== policy.receiptLifetimeMs || + compareSnapshotJournalRevisions(snapshotJournalRevision(retention[0].floor), ceiling) > 0 + ) + return invalid() + const receipts = await k('snapshot_journal_receipts') + .select(k.raw('1 AS occupied')) + .limit(policy.receiptLimit + 1) + if (receipts.length > policy.receiptLimit) return invalid() +} +async function validate( + k: Knex, + p: Plan, + policy: SnapshotJournalReceiptPolicy +): Promise { + const actual = await reserved(k) + if (actual.length !== p.objects.length) return invalid() + for (const expected of p.objects) { + const matches = actual.filter(object => object.name === expected.name && object.type === expected.type) + if (matches.length !== 1 || matches[0].sql !== expected.sql) return invalid() + } + const rows = await k(SNAPSHOT_JOURNAL_SQLITE_GENERATION).select('*').limit(2) + if (rows.length !== 1) return invalid() + const row = rows[0] + if ( + row.id !== 1 || + row.version !== 1 || + typeof row.epoch !== 'string' || + !/^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/.test(row.epoch) || + row.source !== p.source || + ![0, 1].includes(row.complete) + ) + return invalid() + const ceiling = snapshotJournalRevision(row.ceiling) + if (ceiling === '0') return invalid() + const clocks = await k('snapshot_journal_clock') + .select('id', 'enabled', 'reason', k.raw('CAST(revision AS TEXT) revision'), k.raw('CAST(ceiling AS TEXT) ceiling')) + .limit(2) + if ( + clocks.length !== 1 || + clocks[0].id !== 1 || + clocks[0].ceiling !== ceiling || + compareSnapshotJournalRevisions(snapshotJournalRevision(clocks[0].revision), ceiling) > 0 + ) + return invalid() + const clock = clocks[0] + if ( + clock.enabled === 1 + ? clock.reason !== null + : clock.enabled !== 0 || !['capacity-exhausted', 'revision-exhausted', 'key-out-of-range'].includes(clock.reason) + ) + return invalid() + await validateBootstrap(k, row.complete) + await validateRetention(k, policy, ceiling) + return { + epoch: row.epoch, + source: p.source, + ceiling, + complete: row.complete === 1, + enabled: clock.enabled === 1 + } +} + +/** Owned atomic SQLite installation. It does not advertise a reader or register a migration. */ +export async function installSnapshotJournalSqliteGeneration( + k: Knex, + ceiling: SnapshotJournalRevision, + receiptPolicy: SnapshotJournalReceiptPolicy, + config?: Knex.MigratorConfig +): Promise { + const policy = snapshotJournalReceiptPolicy(receiptPolicy) + if (!local(k) || snapshotJournalRevision(ceiling) === '0') return invalid() + return await k.transaction(async t => { + if ((await readGenerationIndexState(t, config)) !== 'v2') return invalid() + await t(indexMetadata) + .where('id', 0) + .update({ complete: t.ref('complete') }) + const p = await plan(t, config), + existing = await reserved(t) + if (existing.length) { + const current = await validate(t, p, policy) + if (current.ceiling !== ceiling) return invalid() + return current + } + await runInSeries(p.objects, async object => { + await t.raw(object.sql) + }) + const epoch = randomUUID() + await t(SNAPSHOT_JOURNAL_SQLITE_GENERATION).insert({ + id: 1, + version: 1, + epoch, + source: p.source, + ceiling, + complete: 0 + }) + await t('snapshot_journal_clock').insert({ + id: 1, + revision: 0, + ceiling, + enabled: 1, + reason: null + }) + await t('snapshot_journal_bootstrap').insert({ id: 1, stream: 0, cursor: null, rowLimit: null, rowsUsed: 0 }) + await t('snapshot_journal_retention').insert({ id: 1, floor: '0', ...policy }) + return await validate(t, p, policy) + }) +} + +/** Validate in the caller's pinned view; migration publication remains a separate prerequisite. */ +export async function readSnapshotJournalSqliteGeneration( + k: Knex, + receiptPolicy: SnapshotJournalReceiptPolicy, + config?: Knex.MigratorConfig +): Promise { + const policy = snapshotJournalReceiptPolicy(receiptPolicy) + return await validate(k, await plan(k, config), policy) +} + +/** Atomic completion cannot be inferred from a caller's last-page acknowledgement. */ +export async function completeSnapshotJournalSqliteGeneration( + k: Knex, + receiptPolicy: SnapshotJournalReceiptPolicy, + config?: Knex.MigratorConfig +): Promise { + const policy = snapshotJournalReceiptPolicy(receiptPolicy) + if (!local(k)) return invalid() + return await k.transaction(async t => { + await t(indexMetadata) + .where('id', 0) + .update({ complete: t.ref('complete') }) + const p = await plan(t, config), + state = await validate(t, p, policy) + const progress = await t('snapshot_journal_bootstrap').select('*').limit(2) + const clock = await t('snapshot_journal_clock').where('id', 1).first('enabled') + if ( + progress.length !== 1 || + progress[0].id !== 1 || + progress[0].stream !== 17 || + progress[0].cursor !== null || + clock.enabled !== 1 + ) + return invalid() + await t(SNAPSHOT_JOURNAL_SQLITE_GENERATION).where('id', 1).update({ complete: 1 }) + return { ...state, complete: true } + }) +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteObservers.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteObservers.ts new file mode 100644 index 000000000..805fe4177 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/journal/SnapshotJournalSqliteObservers.ts @@ -0,0 +1,154 @@ +import type { Knex } from 'knex' +import { WERR_INVALID_OPERATION } from '../../../sdk/WERR_errors' +import { runInSeries } from '../../../utility/runInSeries' +import { numeric } from '../../schema/snapshotSqliteMembership' +import { names } from '../../schema/snapshotSqliteIndexGeneration' +import { + SNAPSHOT_JOURNAL_SQLITE_ADVANCE as tick, + SNAPSHOT_JOURNAL_SQLITE_REVISION as revision, + SNAPSHOT_JOURNAL_SQLITE_WRITABLE as writable +} from './SnapshotJournalSqliteClock' + +const q = (name: string) => '"' + name.replaceAll('"', '""') + '"' +export const snapshotJournalSqliteSources = [ + ...numeric.map(source => source.table), + 'tx_labels_map', + 'output_tags_map', + 'certificate_fields' +] +const scopeKey = 'tableId,userId,id1,id2,exactText' +const physicalKey = 'tableId,id1,id2,exactText' +export const SNAPSHOT_JOURNAL_SQLITE_METADATA_DDL = [ + `CREATE TABLE snapshot_journal_physical(tableId INTEGER NOT NULL,id1 INTEGER NOT NULL,id2 INTEGER NOT NULL,exactText TEXT COLLATE BINARY NOT NULL,revision INTEGER NOT NULL,generation INTEGER NOT NULL,present INTEGER NOT NULL,PRIMARY KEY(${physicalKey}))`, + `CREATE TABLE snapshot_journal_scope(tableId INTEGER NOT NULL,userId INTEGER NOT NULL,id1 INTEGER NOT NULL,id2 INTEGER NOT NULL,exactText TEXT COLLATE BINARY NOT NULL,revision INTEGER NOT NULL,present INTEGER NOT NULL,PRIMARY KEY(${scopeKey}))`, + 'CREATE INDEX snapshot_journal_scope_page ON snapshot_journal_scope(userId,tableId,revision,id1,id2,exactText)', + 'CREATE INDEX snapshot_journal_physical_page ON snapshot_journal_physical(tableId,revision,id1,id2,exactText)' +] +const tuple = (table: string, prefix: string) => { + const key = numeric.find(source => source.table === table)?.key + if (key) return [prefix + '.' + q(key), '0', "''"] + if (table === 'tx_labels_map') return [prefix + '.txLabelId', prefix + '.transactionId', "''"] + if (table === 'output_tags_map') return [prefix + '.outputTagId', prefix + '.outputId', "''"] + return [prefix + '.certificateId', '0', prefix + '.fieldName'] +} +const scopeUpsert = (select: string) => + `INSERT INTO snapshot_journal_scope(${scopeKey},revision,present) ${select} ON CONFLICT(${scopeKey}) DO UPDATE SET revision=excluded.revision,present=excluded.present; ` +const exactWhere = (table: string, prefix: string, alias: string) => { + const left = tuple(table, prefix), + right = tuple(table, alias) + return left.map((key, i) => `CAST(${key} AS BLOB) IS CAST(${right[i]} AS BLOB)`).join(' AND ') +} +/** Native equality admits the declared point index; the byte residual keeps + * generation identity exact even for NOCASE/RTRIM certificate-field keys. */ +const sourceWhere = (table: string, prefix: string, alias: string) => { + const left = tuple(table, prefix), + right = tuple(table, alias) + const indexed = right.flatMap((key, i) => (key.startsWith(alias + '.') ? [`${key}=${left[i]}`] : [])) + return indexed.join(' AND ') + ' AND ' + exactWhere(table, prefix, alias) +} +function keyGuard(key: string[], owner?: string): string { + const [id1, id2, text] = key + const integer = (value: string, minimum: number) => + `(typeof(${value})='integer' AND ${value} BETWEEN ${minimum} AND 9007199254740991)` + const valid = [ + integer(id1, 1), + integer(id2, 0), + `(typeof(${text})='text' AND length(CAST(${text} AS BLOB))<=400)`, + ...(owner ? [integer(owner, 1)] : []) + ].join(' AND ') + return `UPDATE snapshot_journal_clock SET enabled=0,reason='key-out-of-range' WHERE id=1 AND enabled=1 AND NOT (${valid}); ` +} + +function sourceScope(tableId: number, table: string, prefix: string, current: string[]): string { + if (tableId < 8) { + return scopeUpsert( + `SELECT ${tableId},s.userId,s.${q(numeric[tableId].key)},0,'',${revision},1 FROM ${q(table)} s WHERE ${sourceWhere(table, prefix, 's')} AND ${writable}` + ) + } else if (tableId === 10 || tableId === 11) { + return scopeUpsert( + `SELECT ${tableId},snapshotUserId,snapshotLeftId,snapshotRightId,'',${revision},1 FROM ${names.relation} WHERE snapshotTableId=${tableId - 10} AND snapshotLeftId=${current[0]} AND snapshotRightId=${current[1]} AND ${writable}` + ) + } else if (tableId === 12) { + return scopeUpsert( + `SELECT 12,snapshotUserId,snapshotCertificateId,0,snapshotFieldName,${revision},1 FROM ${names.certificate} WHERE snapshotCertificateId=${current[0]} AND snapshotFieldName=${current[2]} AND ${writable}` + ) + } + return '' +} +function freshGeneration(event: string, table: string): string { + if (event === 'INSERT') return '1' + if (event === 'UPDATE') return `NOT (${exactWhere(table, 'OLD', 'NEW')})` + return '0' +} +function physicalTrigger(tableId: number, table: string, event: string, changed: string): string { + const prefix = event === 'DELETE' ? 'OLD' : 'NEW', + current = tuple(table, prefix) + const regenerate = freshGeneration(event, table) + const writePhysical = (p: string, fresh: string) => + `INSERT INTO snapshot_journal_physical(${physicalKey},revision,generation,present) SELECT ${tableId},${tuple(table, p).join(',')},${revision},${revision},EXISTS(SELECT 1 FROM ${q(table)} s WHERE ${sourceWhere(table, p, 's')}) WHERE ${writable} ON CONFLICT(${physicalKey}) DO UPDATE SET revision=excluded.revision,generation=CASE WHEN ${fresh} THEN excluded.generation ELSE snapshot_journal_physical.generation END,present=excluded.present; ` + let body = keyGuard(current, tableId < 8 ? prefix + '.userId' : undefined) + if (event === 'UPDATE') body += keyGuard(tuple(table, 'OLD'), tableId < 8 ? 'OLD.userId' : undefined) + body += tick + if (event === 'UPDATE') body += writePhysical('OLD', '0') + body += writePhysical(prefix, regenerate) + body += sourceScope(tableId, table, prefix, current) + return `CREATE TRIGGER snapshot_journal_physical_${tableId}_${event} AFTER ${event} ON ${q(table)} ${event === 'UPDATE' ? 'WHEN ' + changed : ''} BEGIN ${body} END` +} + +/** Prepare observers only after the caller validates the completed v2 source generation. */ +export async function snapshotJournalSqliteObserverSql(k: Knex): Promise { + if (!['sqlite3', 'better-sqlite3'].includes(k.client.config.client)) + throw new WERR_INVALID_OPERATION('Snapshot journal observers require SQLite') + const definitions: string[] = [] + const membership: Array<{ + table: string + expressions: (p: string) => string[] + present: (p: string) => string + }> = [ + { + table: names.profile, + expressions: p => [p + '.snapshotTableId', p + '.snapshotUserId', p + '.snapshotRowId', '0', "''"], + present: () => '1' + }, + { + table: names.relation, + expressions: p => [ + p + '.snapshotTableId+10', + p + '.snapshotUserId', + p + '.snapshotLeftId', + p + '.snapshotRightId', + "''" + ], + present: () => '1' + }, + { + table: names.certificate, + expressions: p => ['12', p + '.snapshotUserId', p + '.snapshotCertificateId', '0', p + '.snapshotFieldName'], + present: () => '1' + }, + { + table: names.keys, + expressions: p => ['CASE ' + p + '.tableId WHEN 0 THEN 9 ELSE 8 END', p + '.userId', p + '.rowId', '0', "''"], + present: p => p + '.present' + } + ] + for (const [i, source] of membership.entries()) + for (const event of ['INSERT', 'DELETE', 'UPDATE']) { + const p = event === 'DELETE' ? 'OLD' : 'NEW', + fields = source.expressions(p) + const selected = [...fields, revision, event === 'DELETE' ? '0' : source.present(p)].join(',') + definitions.push( + `CREATE TRIGGER snapshot_journal_scope_${i}_${event} AFTER ${event} ON ${q(source.table)} BEGIN ${keyGuard(fields.slice(2), fields[1])}${tick}${scopeUpsert('SELECT ' + selected + ' WHERE ' + writable)} END` + ) + } + await runInSeries(snapshotJournalSqliteSources.entries(), async ([tableId, table]) => { + const columns: Array<{ name: string }> = await k.raw('PRAGMA table_info(??)', [table]) + if (columns.length === 0) throw new WERR_INVALID_OPERATION('Missing snapshot journal source') + const changed = columns + .map(({ name }) => `CAST(OLD.${q(name)} AS BLOB) IS NOT CAST(NEW.${q(name)} AS BLOB)`) + .join(' OR ') + for (const event of ['INSERT', 'UPDATE', 'DELETE']) + definitions.push(physicalTrigger(tableId, table, event, changed)) + }) + return definitions +} diff --git a/packages/wallet/wallet-toolbox/src/storage/snapshot/runSnapshotSyncSession.ts b/packages/wallet/wallet-toolbox/src/storage/snapshot/runSnapshotSyncSession.ts new file mode 100644 index 000000000..86d51e939 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/snapshot/runSnapshotSyncSession.ts @@ -0,0 +1,189 @@ +import { SnapshotResourceLimitError } from './SnapshotResourceLimitError' +import type { RequestSyncChunkArgs, SyncChunk } from '../../sdk/WalletStorage.interfaces' +import { WERR_INVALID_OPERATION, WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +import { SyncPageBudget } from '../sync/SyncPageBudget' +import type { SyncSessionOptions, SyncSessionProgress, SyncSessionResult } from '../sync/syncSession' +import { snapshotSyncTables, type SnapshotSyncCheckpoint, type SnapshotSyncStorage } from './SnapshotSync' +import type { WalletReadSnapshot, WalletSnapshotCursor } from './WalletReadSnapshot' +import { copySnapshotCursor, sameSnapshotArchivePosition } from './SnapshotCursor' + +interface SnapshotSession { + view: WalletReadSnapshot + destination: SnapshotSyncStorage + activeStorage: string + /** Check primary generation and cancellation inside background write admission. */ + commit: (operation: () => Promise) => Promise +} + +function copyCheckpoint(value: SnapshotSyncCheckpoint): SnapshotSyncCheckpoint +function copyCheckpoint(value: SnapshotSyncCheckpoint | undefined): SnapshotSyncCheckpoint | undefined +function copyCheckpoint(value: SnapshotSyncCheckpoint | undefined): SnapshotSyncCheckpoint | undefined { + return value === undefined ? undefined : { ...value, cursor: copySnapshotCursor(value.cursor) } +} + +type NotifyProgress = (state: SyncSessionProgress['state'], timing?: Partial) => void + +function validateAcknowledgement( + previous: SnapshotSyncCheckpoint, + page: { done: boolean; cursor?: WalletSnapshotCursor }, + next: SnapshotSyncCheckpoint +): void { + const tableIndex = previous.tableIndex + Number(page.done) + const expected = { + ...previous, + sequence: previous.sequence + 1, + tableIndex, + done: tableIndex === snapshotSyncTables.length + } + const bindings = [ + 'version', + 'sessionId', + 'identityKey', + 'sourceStorageIdentityKey', + 'destinationStorageIdentityKey', + 'snapshotId', + 'sequence', + 'tableIndex', + 'done' + ] as const + const cursor = page.done ? undefined : page.cursor + if ( + bindings.some(key => next[key] !== expected[key]) || + next.cursor?.version !== cursor?.version || + next.cursor?.snapshotId !== cursor?.snapshotId || + next.cursor?.table !== cursor?.table || + JSON.stringify(next.cursor?.after) !== JSON.stringify(cursor?.after) || + !sameSnapshotArchivePosition(next.cursor?.archivePosition, cursor?.archivePosition) + ) { + throw new WERR_INVALID_OPERATION( + 'Snapshot destination acknowledgement does not match the committed page; reload its durable checkpoint' + ) + } +} + +async function preparePage( + session: SnapshotSession, + checkpoint: SnapshotSyncCheckpoint, + limits: { maxRows: number; maxBytes: number }, + notify: NotifyProgress, + cancelled: () => boolean +) { + notify('reading') + if (cancelled()) return undefined + const readAt = Date.now() + const page = await session.view.readPage(snapshotSyncTables[checkpoint.tableIndex], checkpoint.cursor, limits) + const readMs = Date.now() - readAt + if (cancelled()) return undefined + notify('preparing', { readMs }) + if (cancelled()) return undefined + const prepareAt = Date.now() + const apply = await session.destination.prepare(checkpoint, page) + const prepareMs = Date.now() - prepareAt + if (cancelled()) return undefined + return { page, apply, readMs, prepareMs } +} + +async function rejectClosedSnapshot(view: WalletReadSnapshot): Promise { + // Physical cleanup may need database I/O; await it after queue ownership + // was released. Its actual expiry/cancellation/read error stays authoritative. + await view.closed + if (Date.now() >= view.expiresAt) + throw new SnapshotResourceLimitError('Snapshot source expired before destination commit') + throw new WERR_INVALID_OPERATION('Snapshot source closed before destination commit') +} + +/** One packed page in flight. The caller owns source cleanup, including error/cancellation. */ +export async function runSnapshotSyncSession( + session: SnapshotSession, + options: SyncSessionOptions +): Promise { + const maxItems = options.maxItems ?? 1000 + const maxBytes = options.maxRoughSize ?? 262144 + if ( + !Number.isSafeInteger(maxItems) || + maxItems < 1 || + maxItems > 1000 || + !Number.isSafeInteger(maxBytes) || + maxBytes < 1 || + maxBytes > 10000000 + ) { + throw new WERR_INVALID_PARAMETER('limits', '1–1000 rows and 1–10000000 payload bytes') + } + const result: SyncSessionResult = { status: 'completed', mode: 'paged', pages: 0, inserts: 0, updates: 0 } + const notify = (state: SyncSessionProgress['state'], timing: Partial = {}): void => { + options.onProgress?.({ ...result, ...timing, state, snapshotCheckpoint: copyCheckpoint(result.snapshotCheckpoint) }) + } + const cancelled = (): boolean => { + if (options.signal?.aborted !== true) return false + result.status = 'cancelled' + notify('cancelling') + notify('cancelled') + return true + } + if (cancelled()) return result + const start = await session.commit(async () => + options.signal?.aborted === true ? undefined : await session.destination.begin(session.view, session.activeStorage) + ) + if (start === undefined) { + cancelled() + return result + } + result.snapshotCheckpoint = start + const budget = new SyncPageBudget() + const args: RequestSyncChunkArgs = { + identityKey: start.identityKey, + fromStorageIdentityKey: start.sourceStorageIdentityKey, + toStorageIdentityKey: start.destinationStorageIdentityKey, + maxItems, + maxRoughSize: maxBytes, + offsets: [] + } + while (!result.snapshotCheckpoint.done) { + if (cancelled()) return result + const checkpoint = result.snapshotCheckpoint + const table = snapshotSyncTables[checkpoint.tableIndex] + const prepared = await preparePage( + session, + checkpoint, + { maxRows: budget.apply(args).maxItems, maxBytes }, + notify, + cancelled + ) + if (prepared === undefined) return result + const { page, apply, readMs, prepareMs } = prepared + notify('committing', { readMs, prepareMs }) + const queuedAt = Date.now() + let commitAt = queuedAt + const committed = await session.commit(async () => { + if (options.signal?.aborted === true) return undefined + if (!session.view.isOpen) return 'source-closed' as const + commitAt = Date.now() + return await apply() + }) + if (committed === 'source-closed') return await rejectClosedSnapshot(session.view) + if (committed === undefined) { + cancelled() + return result + } + validateAcknowledgement(checkpoint, page, committed.checkpoint) + const commitMs = Date.now() - commitAt + result.pages++ + result.inserts += committed.inserts + result.updates += committed.updates + result.snapshotCheckpoint = copyCheckpoint(committed.checkpoint) + budget.committed( + { + userIdentityKey: start.identityKey, + fromStorageIdentityKey: start.sourceStorageIdentityKey, + toStorageIdentityKey: start.destinationStorageIdentityKey, + [table]: page.rows + } as SyncChunk, + readMs + prepareMs + commitMs, + readMs + prepareMs + ) + notify('committed', { readMs, prepareMs, queueMs: commitAt - queuedAt, commitMs }) + if (cancelled()) return result + } + notify('completed') + return result +} diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/StorageAccessQueue.test.ts b/packages/wallet/wallet-toolbox/src/storage/sync/StorageAccessQueue.test.ts new file mode 100644 index 000000000..aeb132dce --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/StorageAccessQueue.test.ts @@ -0,0 +1,131 @@ +import { StorageAccessQueue } from './StorageAccessQueue' + +test('shares bounded reads, fences writers and does not let later readers starve a writer', async () => { + const queue = new StorageAccessQueue() + const readers = await Promise.all(Array.from({ length: 8 }, async () => await queue.acquire('read'))) + let writerStarted = false + const writer = queue.acquire('exclusive').then(release => { + writerStarted = true + return release + }) + let laterStarted = false + const later = queue.acquire('read').then(release => { + laterStarted = true + return release + }) + await Promise.resolve() + expect(writerStarted).toBe(false) + for (const release of readers.slice(0, 7)) release() + await Promise.resolve() + expect(writerStarted).toBe(false) + readers[7]() + const releaseWriter = await writer + expect(laterStarted).toBe(false) + releaseWriter() + const releaseLater = await later + expect(laterStarted).toBe(true) + releaseLater() + // A duplicate completion cannot release another operation's ownership. + releaseWriter() + releaseLater() + const next = await queue.acquire('exclusive') + next() +}) + +test('admits at most eight concurrent readers', async () => { + const queue = new StorageAccessQueue() + const readers = await Promise.all(Array.from({ length: 8 }, async () => await queue.acquire('read'))) + let started = false + const ninth = queue.acquire('read').then(release => { + started = true + return release + }) + await Promise.resolve() + expect(started).toBe(false) + readers[0]() + const release = await ninth + for (const done of readers) done() + release() +}) + +test('prefers foreground work while guaranteeing a waiting background page within eight grants', async () => { + const queue = new StorageAccessQueue() + const unblock = await queue.acquire('exclusive') + const order: string[] = [] + const background = queue.acquire('exclusive', 'background').then(release => { + order.push('background') + release() + }) + const foreground = Array.from({ length: 20 }, (_, index) => + queue.acquire('exclusive').then(release => { + order.push(`foreground ${index}`) + release() + }) + ) + unblock() + await Promise.all([background, ...foreground]) + expect(order[0]).toBe('foreground 0') + expect(order.indexOf('background')).toBeLessThanOrEqual(8) + expect(order.filter(item => item !== 'background')).toEqual( + Array.from({ length: 20 }, (_, index) => `foreground ${index}`) + ) +}) + +test('ages a background page ahead of newly queued foreground work', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1000) + try { + const queue = new StorageAccessQueue() + const unblock = await queue.acquire('exclusive') + const order: string[] = [] + const background = queue.acquire('exclusive', 'background').then(release => { + order.push('background') + release() + }) + now.mockReturnValue(2001) + const foreground = queue.acquire('exclusive').then(release => { + order.push('foreground') + release() + }) + unblock() + await Promise.all([background, foreground]) + expect(order).toEqual(['background', 'foreground']) + } finally { + now.mockRestore() + } +}) + +test.each(['foreground', 'background'] as const)('retains FIFO order with only %s waiters', async priority => { + const queue = new StorageAccessQueue() + const unblock = await queue.acquire('exclusive') + const order: number[] = [] + const waiting = Array.from({ length: 12 }, (_, index) => + queue.acquire('exclusive', priority).then(release => { + order.push(index) + release() + }) + ) + unblock() + await Promise.all(waiting) + expect(order).toEqual(Array.from({ length: 12 }, (_, index) => index)) +}) + +test('finds an aged background waiter behind the foreground head', async () => { + const now = jest.spyOn(Date, 'now').mockReturnValue(1000) + try { + const queue = new StorageAccessQueue() + const unblock = await queue.acquire('exclusive') + const order: string[] = [] + const waiting = (['foreground', 'background', 'foreground'] as const).map((priority, index) => + queue.acquire('exclusive', priority).then(release => { + order.push(`${priority} ${index}`) + release() + }) + ) + now.mockReturnValue(2000) + unblock() + await Promise.all(waiting) + expect(order).toEqual(['background 1', 'foreground 0', 'foreground 2']) + } finally { + now.mockRestore() + } +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/StorageAccessQueue.ts b/packages/wallet/wallet-toolbox/src/storage/sync/StorageAccessQueue.ts new file mode 100644 index 000000000..48d39611b --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/StorageAccessQueue.ts @@ -0,0 +1,63 @@ +type Mode = 'read' | 'exclusive' +type Priority = 'foreground' | 'background' +interface Waiter { + mode: Mode + priority: Priority + queuedAt: number + grant: (release: () => void) => void +} + +/** + * One ownership queue: exclusive operations never overlap; opted-in reads + * share up to eight slots. Foreground work may pass a background page at most + * eight times or for one second. FIFO order is retained within each priority. + */ +export class StorageAccessQueue { + private readonly waiters: Waiter[] = [] + private readers = 0 + private exclusive = false + private foregroundGrants = 0 + + acquire(mode: Mode, priority: Priority = 'foreground'): Promise<() => void> { + return new Promise(resolve => { + this.waiters.push({ mode, priority, queuedAt: Date.now(), grant: resolve }) + this.pump() + }) + } + + private first(priority: Priority): number { + for (let index = 0; index < this.waiters.length; index++) { + if (this.waiters[index].priority === priority) return index + } + return -1 + } + + private nextIndex(): number { + const background = this.first('background') + if (background >= 0 && (this.foregroundGrants >= 8 || Date.now() - this.waiters[background].queuedAt >= 1000)) + return background + const foreground = this.first('foreground') + return Math.max(foreground, 0) + } + + private pump(): void { + while (!this.exclusive && this.waiters.length > 0) { + // Uncontended access needs no priority search or predicate allocation. + const index = this.waiters.length === 1 ? 0 : this.nextIndex() + const waiter = this.waiters[index] + if (waiter.mode === 'exclusive' ? this.readers > 0 : this.readers >= 8) return + this.waiters.splice(index, 1) + if (waiter.mode === 'exclusive') this.exclusive = true + else this.readers++ + this.foregroundGrants = waiter.priority === 'background' ? 0 : Math.min(8, this.foregroundGrants + 1) + let released = false + waiter.grant(() => { + if (released) return + released = true + if (waiter.mode === 'exclusive') this.exclusive = false + else this.readers-- + this.pump() + }) + } + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.test.ts b/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.test.ts index f3e7eefb7..6030ed029 100644 --- a/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.test.ts @@ -1,9 +1,15 @@ import { SyncPageBudget } from './SyncPageBudget' import type { RequestSyncChunkArgs, SyncChunk } from '../../sdk/WalletStorage.interfaces' -const args = { maxItems: 1000, maxRoughSize: 2000000, offsets: [{ name: 'provenTx', offset: 700 }] } as RequestSyncChunkArgs +const args = { + maxItems: 1000, + maxRoughSize: 2000000, + offsets: [{ name: 'provenTx', offset: 700 }] +} as RequestSyncChunkArgs const chunk = (count: number, proof = false): SyncChunk => ({ - fromStorageIdentityKey: 'from', toStorageIdentityKey: 'to', userIdentityKey: 'user', + fromStorageIdentityKey: 'from', + toStorageIdentityKey: 'to', + userIdentityKey: 'user', [proof ? 'provenTxs' : 'transactions']: Array.from({ length: count }, () => ({})) }) @@ -37,3 +43,56 @@ test('ignores unusable timings and empty pages and starts each copy independentl expect(budget.apply(args).maxItems).toBe(128) expect(new SyncPageBudget().apply(args).maxItems).toBe(64) }) + +test.each([false, true])('amortizes fixed latency without collapsing to single rows (proofs=%s)', proof => { + const budget = new SyncPageBudget() + const limits: number[] = [] + let remaining = 10000 + let pages = 0 + while (remaining > 0) { + const limit = budget.apply(args).maxItems + limits.push(limit) + const count = Math.min(limit, remaining) + // Independent fixed costs in the source request and destination commit. + budget.committed(chunk(count, proof), 20000 + count * 2, 15000 + count) + remaining -= count + pages++ + expect(pages).toBeLessThan(200) + } + expect(Math.min(...limits)).toBeGreaterThan(1) + expect(limits.slice(-5).every(limit => limit === (proof ? 128 : 1000))).toBe(true) +}) + +test('distinguishes per-proof work from fixed overhead and responds to a slowdown', () => { + const budget = new SyncPageBudget() + for (let i = 0; i < 10; i++) { + const count = budget.apply(args).maxItems + budget.committed(chunk(count, true), 16000 + count * 100, 15000 + count * 20) + } + expect(budget.apply(args).maxItems).toBeGreaterThanOrEqual(49) + expect(budget.apply(args).maxItems).toBeLessThanOrEqual(51) + const before = budget.apply(args).maxItems + budget.committed(chunk(before, true), 16000 + before * 1000, 15000 + before * 20) + expect(budget.apply(args).maxItems).toBeLessThan(before / 2) +}) + +test('recovers from the single-row floor using bounded upward probes', () => { + const budget = new SyncPageBudget() + budget.committed(chunk(1, true), 20000, 19000) + const limits: number[] = [] + for (let i = 0; i < 30; i++) { + const count = budget.apply(args).maxItems + limits.push(count) + budget.committed(chunk(count, true), 20000, 19000) + } + expect(limits[0]).toBe(1) + expect(limits.slice(0, 4)).toContain(2) + expect(limits.slice(-5)).toEqual([128, 128, 128, 128, 128]) +}) + +test('does not reuse cheap metadata estimates for expensive proofs', () => { + const budget = new SyncPageBudget() + for (let i = 0; i < 10; i++) budget.committed(chunk(budget.apply(args).maxItems), 20000, 19000) + budget.committed(chunk(100, true), 50000, 1000) + expect(budget.apply(args).maxItems).toBeLessThanOrEqual(10) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.ts b/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.ts index e03eae666..78d29ae2b 100644 --- a/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.ts +++ b/packages/wallet/wallet-toolbox/src/storage/sync/SyncPageBudget.ts @@ -1,22 +1,95 @@ import type { RequestSyncChunkArgs, SyncChunk } from '../../sdk/WalletStorage.interfaces' +interface PageCost { + records: number + readMs: number + commitMs: number +} + +/** Fit fixed overhead separately from marginal work, using bounded recent history. */ +function marginalCost( + samples: PageCost[], + phase: 'readMs' | 'commitMs', + previousFixedMs: number +): { perRecordMs: number; fixedMs: number } { + let totalRecords = 0 + let totalMs = 0 + for (const sample of samples) { + totalRecords += sample.records + totalMs += sample[phase] + } + const averageRecords = totalRecords / samples.length + const averageMs = totalMs / samples.length + let variance = 0 + let covariance = 0 + for (const sample of samples) { + const delta = sample.records - averageRecords + variance += delta ** 2 + covariance += delta * (sample[phase] - averageMs) + } + // Until page sizes differ there is no evidence that any cost is fixed. + if (variance === 0) { + const fixedMs = Math.min(previousFixedMs, averageMs) + return { perRecordMs: (averageMs - fixedMs) / averageRecords, fixedMs } + } + const slope = Math.max(0, covariance / variance) + const fixedMs = Math.max(0, averageMs - slope * averageRecords) + const latest = samples.at(-1)! + // React to a newly expensive page immediately, even when the rolling fit + // still contains cheap pages. Never subtract more than its observed cost. + return { perRecordMs: Math.max(slope, (latest[phase] - fixedMs) / latest.records), fixedMs } +} + /** Per-copy work budget. Bytes alone cannot bound network-backed proof checks. */ export class SyncPageBudget { private maxItems = 64 + private samples: PageCost[] = [] + private proofs?: boolean + private fixedReadMs = 0 + private fixedCommitMs = 0 + private pagesSinceProbe = 0 apply(args: RequestSyncChunkArgs): RequestSyncChunkArgs { return { ...args, maxItems: Math.min(args.maxItems, this.maxItems) } } - committed(chunk: SyncChunk, elapsedMs: number): void { - if (!Number.isFinite(elapsedMs) || elapsedMs < 0) return - const records = Object.values(chunk).reduce((count, value) => - count + (Array.isArray(value) ? value.length : 0), 0) + /** + * `elapsedMs` includes read and commit; an optional read measurement keeps + * network overhead separate from destination work. Old callers remain valid. + */ + committed(chunk: SyncChunk, elapsedMs: number, readMs = 0): void { + if (!Number.isFinite(elapsedMs) || elapsedMs < 0 || !Number.isFinite(readMs) || readMs < 0 || readMs > elapsedMs) + return + let records = 0 + for (const value of Object.values(chunk)) { + if (Array.isArray(value)) records += value.length + } if (records === 0) return - // Leave headroom under the authentication deadline and limit growth when - // moving from cheap metadata to network-backed proof validation. - const ceiling = (chunk.provenTxs?.length ?? 0) > 0 ? 128 : 1000 - const suggested = Math.floor(records * 5000 / Math.max(1, elapsedMs)) - this.maxItems = Math.max(1, Math.min(ceiling, this.maxItems * 2, suggested)) + const proofs = (chunk.provenTxs?.length ?? 0) > 0 + // Metadata throughput does not predict network-backed proof checks. + if (proofs !== this.proofs) { + this.samples = [] + this.fixedReadMs = 0 + this.fixedCommitMs = 0 + this.pagesSinceProbe = 0 + } + this.proofs = proofs + this.samples.push({ records, readMs, commitMs: elapsedMs - readMs }) + if (this.samples.length > 6) this.samples.shift() + const ceiling = proofs ? 128 : 1000 + const read = marginalCost(this.samples, 'readMs', this.fixedReadMs) + const commit = marginalCost(this.samples, 'commitMs', this.fixedCommitMs) + this.fixedReadMs = read.fixedMs + this.fixedCommitMs = commit.fixedMs + const perRecordMs = read.perRecordMs + commit.perRecordMs + const suggested = Math.floor(5000 / Math.max(0.001, perRecordMs)) + let next = Math.max(1, Math.min(ceiling, this.maxItems * 2, suggested)) + this.pagesSinceProbe++ + // At the one-record floor there is no size variation from which to learn + // fixed latency. A bounded two-record probe prevents permanent collapse. + // A genuinely expensive proof returns to one on the next measurement. + if (next === 1 && this.maxItems === 1 && this.pagesSinceProbe >= 4) next = 2 + if (next > this.maxItems) this.pagesSinceProbe = 0 + this.maxItems = next } } diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/snapshotSyncPage.test.ts b/packages/wallet/wallet-toolbox/src/storage/sync/snapshotSyncPage.test.ts new file mode 100644 index 000000000..6d704df42 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/snapshotSyncPage.test.ts @@ -0,0 +1,45 @@ +import type { RequestSyncChunkArgs, SyncChunk } from '../../sdk/WalletStorage.interfaces' +import { snapshotSyncPage } from './snapshotSyncPage' + +test.each(['array', 'typed view', 'Buffer view'] as const)( + 'owns %s bytes and checkpoint values across an asynchronous handoff', + async representation => { + const backing = representation === 'Buffer view' ? Buffer.from([9, 1, 2, 3, 8]) : new Uint8Array([9, 1, 2, 3, 8]) + const bytes = representation === 'array' ? [1, 2, 3] : backing.subarray(1, 4) + const since = new Date(10) + const updatedAt = new Date(20) + const args = { + identityKey: 'wallet', + fromStorageIdentityKey: 'source', + toStorageIdentityKey: 'destination', + maxItems: 1, + maxRoughSize: 1024, + since, + offsets: [{ name: 'transaction', offset: 1 }] + } as RequestSyncChunkArgs + // The snapshot owns incoming byte storage before downstream schema/proof + // validation. Runtime byte views must never keep a producer's backing store. + const chunk = { + fromStorageIdentityKey: 'source', + toStorageIdentityKey: 'destination', + userIdentityKey: 'wallet', + transactions: [{ rawTx: bytes, updated_at: updatedAt }] + } as unknown as SyncChunk + const snapshot = snapshotSyncPage(args, chunk) + await Promise.resolve() + bytes[0] = 7 + since.setTime(30) + updatedAt.setTime(40) + args.offsets[0].offset = 99 + + const copied = snapshot.chunk.transactions![0] + expect(Array.from(copied.rawTx!)).toEqual([1, 2, 3]) + expect(copied.updated_at.getTime()).toBe(20) + expect(snapshot.args.since?.getTime()).toBe(10) + expect(snapshot.args.offsets[0].offset).toBe(1) + copied.rawTx![1] = 6 + expect(bytes[1]).toBe(2) + expect(backing[0]).toBe(9) + expect(backing[4]).toBe(8) + } +) diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/snapshotSyncPage.ts b/packages/wallet/wallet-toolbox/src/storage/sync/snapshotSyncPage.ts new file mode 100644 index 000000000..57a7576ad --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/snapshotSyncPage.ts @@ -0,0 +1,44 @@ +import type { RequestSyncChunkArgs, SyncChunk } from '../../sdk/WalletStorage.interfaces' +import { WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' + +/** Detach flat sync records, dates and byte arrays without a JSON-sized intermediate string. */ +function recordCopy(record: T): T { + const result = { ...record } + for (const key of Object.keys(result) as Array) { + const value = result[key] + let copy: unknown = value + if (Array.isArray(value)) copy = value.slice() + else if (value instanceof Uint8Array) copy = new Uint8Array(value) + else if (value != null && typeof value === 'object') { + // Date's internal-slot check also accepts dates returned by an IndexedDB + // implementation in another realm; arbitrary nested objects are invalid. + try { + copy = new Date(Date.prototype.getTime.call(value)) + } catch { + throw new WERR_INVALID_PARAMETER('chunk', 'flat table records with dates and byte arrays') + } + } + result[key] = copy as T[keyof T] + } + return result +} + +export function snapshotSyncPage( + args: RequestSyncChunkArgs, + chunk: SyncChunk +): { args: RequestSyncChunkArgs; chunk: SyncChunk } { + const snapshot = { ...chunk } + for (const key of Object.keys(snapshot) as Array) { + const value = snapshot[key] + if (Array.isArray(value)) Object.assign(snapshot, { [key]: value.map(recordCopy) }) + } + if (chunk.user != null) snapshot.user = recordCopy(chunk.user) + return { + args: { + ...args, + since: args.since == null ? undefined : new Date(args.since), + offsets: args.offsets.map(offset => ({ ...offset })) + }, + chunk: snapshot + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.test.ts b/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.test.ts index c81af5599..33ce0fa82 100644 --- a/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.test.ts +++ b/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.test.ts @@ -4,13 +4,15 @@ import { once } from 'node:events' import { StorageServer } from '../remoting/StorageServer' import { _tu } from '../../../test/utils/TestUtilsWalletStorage' import { PrivateKey } from '@bsv/sdk' +import { WalletError } from '../../sdk/WalletError' +import { WERR_NETWORK_CHAIN } from '../../sdk/WERR_errors' import { StorageIdb } from '../StorageIdb' import { StorageProvider } from '../StorageProvider' import { WalletStorageManager } from '../WalletStorageManager' import { EntitySyncState } from '../schema/entities/EntitySyncState' import { StorageClient } from '../remoting/StorageClient' import { validateSyncCheckpoint } from './syncCheckpoint' -import type { RequestSyncChunkArgs, SyncCheckpoint } from '../../sdk/WalletStorage.interfaces' +import type { ProcessSyncChunkResult, RequestSyncChunkArgs, SyncCheckpoint } from '../../sdk/WalletStorage.interfaces' async function makeStorage(): Promise { const storage = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) @@ -51,6 +53,84 @@ describe('compact sync checkpoints', () => { expect(() => validateSyncCheckpoint(c, { since: new Date() })).toThrow() }) + test.each([undefined, new Date('2026-01-01T00:00:00.000Z')])( + 'accepts a complete terminal reset with unchanged since %s and preserves other progress guards', + since => { + const reset = { ...checkpoint(), since } + const previous = { ...reset, offsets: reset.offsets.map((entry, index) => ({ ...entry, offset: index + 1 })) } + expect(validateSyncCheckpoint(reset, previous, true)).toEqual(reset) + expect(() => validateSyncCheckpoint(reset, previous)).toThrow('Invalid sync checkpoint') + expect(() => validateSyncCheckpoint(reset, previous, false)).toThrow('Invalid sync checkpoint') + expect(() => validateSyncCheckpoint({ ...reset, syncStateId: 2 }, previous, true)).toThrow() + expect(() => validateSyncCheckpoint({ ...reset, offsets: [] }, previous, true)).toThrow() + for (const offset of [-1, 0.5, Number.MAX_SAFE_INTEGER + 1]) { + const invalid = { + ...reset, + offsets: reset.offsets.map((entry, index) => (index === 0 ? { ...entry, offset } : entry)) + } + expect(() => validateSyncCheckpoint(invalid, previous, true)).toThrow() + } + const partial = { + ...reset, + offsets: reset.offsets.map((entry, index) => (index === 0 ? { ...entry, offset: 1 } : entry)) + } + expect(() => validateSyncCheckpoint(partial, previous, true)).toThrow('Invalid sync checkpoint') + const advanced = { ...previous, offsets: previous.offsets.map(entry => ({ ...entry, offset: entry.offset + 1 })) } + expect(validateSyncCheckpoint(advanced, previous, true)).toEqual(advanced) + if (since != null) { + expect(() => validateSyncCheckpoint({ ...reset, since: undefined }, previous, true)).toThrow() + expect(() => + validateSyncCheckpoint({ ...reset, since: new Date(since.getTime() - 1) }, previous, true) + ).toThrow() + } + } + ) + + test.each([false, undefined, 'true', 1])('rejects an offset reset without literal done: true (%s)', async done => { + const reset = { ...checkpoint(), since: new Date('2026-01-01T00:00:00.000Z') } + const previous = { ...reset, offsets: reset.offsets.map(entry => ({ ...entry, offset: 1 })) } + const client = new StorageClient({} as never, 'https://storage.example') + const rpc = jest.spyOn(client as never, 'rpcCall' as never) as jest.SpyInstance + rpc.mockResolvedValue({ done, inserts: 0, updates: 0, nextCheckpoint: reset }) + await expect(client.processSyncChunk(previous as RequestSyncChunkArgs, {} as never)).rejects.toThrow( + 'Invalid sync checkpoint' + ) + expect(rpc).toHaveBeenCalledTimes(1) + }) + + test.each(['foreign state', 'earlier timestamp', 'partial reset'])( + 'does not let a remote terminal reply bypass %s validation', + async invalid => { + const reset = { ...checkpoint(), since: new Date('2026-01-01T00:00:00.000Z') } + const previous = { ...reset, offsets: reset.offsets.map(entry => ({ ...entry, offset: 2 })) } + if (invalid === 'foreign state') reset.syncStateId = 2 + if (invalid === 'earlier timestamp') reset.since = new Date(reset.since.getTime() - 1) + if (invalid === 'partial reset') reset.offsets[0].offset = 1 + const client = new StorageClient({} as never, 'https://storage.example') + const rpc = jest.spyOn(client as never, 'rpcCall' as never) as jest.SpyInstance + rpc.mockResolvedValue({ done: true, inserts: 0, updates: 0, nextCheckpoint: reset }) + await expect(client.processSyncChunk(previous as RequestSyncChunkArgs, {} as never)).rejects.toThrow( + 'Invalid sync checkpoint' + ) + expect(rpc).toHaveBeenCalledTimes(1) + } + ) + + test.each([false, true])( + 'forwards a returned failure before validating its checkpoint (serialized: %s)', + async serialized => { + const original = new WERR_NETWORK_CHAIN('synthetic provider failure') + const error = serialized ? JSON.parse(WalletError.unknownToJson(original)) : original + const result = { error, done: true, inserts: 999, updates: 999, nextCheckpoint: { syncStateId: -1 } } + const client = new StorageClient({} as never, 'https://storage.example') + const rpc = jest.spyOn(client as never, 'rpcCall' as never) as jest.SpyInstance + rpc.mockResolvedValue(result) + await expect(client.processSyncChunk(checkpoint() as RequestSyncChunkArgs, {} as never)).resolves.toBe(result) + expect(result.error).toBe(error) + expect(rpc).toHaveBeenCalledTimes(1) + } + ) + test('checkpoint size stays bounded as the durable ID map grows and remains user scoped', async () => { const storage = await makeStorage() const identityKey = PrivateKey.fromRandom().toPublicKey().toString() @@ -190,77 +270,122 @@ describe('compact sync checkpoints', () => { } ) - test('backs up and restores every page through authenticated HTTP with compact committed progress', async () => { - const remote = await _tu.createSQLiteTestWallet({ databaseName: 'compactCheckpointHttp', dropAll: true }) - const source = await makeStorage() - const restored = await makeStorage() - const server = new StorageServer(remote.activeStorage, { - port: 0, - wallet: remote.wallet, - monetize: false, - logRpcRequests: false, - calculateRequestPrice: async () => 0 - }) - let client: StorageClient | undefined - try { - server.start() - if (!server.server.listening) await once(server.server, 'listening') - const address = server.server.address() - if (address == null || typeof address === 'string') throw new Error('test server did not bind') - client = new StorageClient(remote.wallet, `http://localhost:${address.port}`, { binaryRequests: true }) - const identityKey = remote.identityKey - const manager = new WalletStorageManager(identityKey, source) - await manager.makeAvailable() - const { user } = await source.findOrInsertUser(identityKey) - for (let i = 0; i < 37; i++) await source.findOrInsertTxLabel(user.userId, `http fixture ${i}`) - const read = source.getSyncChunk.bind(source) - jest.spyOn(source, 'getSyncChunk').mockImplementation(args => read({ ...args, maxItems: 5 })) - const checkpoints = jest.spyOn(client, 'getSyncCheckpoint') - const fullStates = jest.spyOn(client, 'findOrInsertSyncStateAuth') - const backup = await manager.syncToWriter({ identityKey }, client) - expect(backup.inserts).toBeGreaterThanOrEqual(37) - expect(checkpoints).toHaveBeenCalledTimes(1) - expect(fullStates).not.toHaveBeenCalled() - const remoteRead = remote.activeStorage.getSyncChunk.bind(remote.activeStorage) - jest.spyOn(remote.activeStorage, 'getSyncChunk').mockImplementation(args => remoteRead({ ...args, maxItems: 5 })) - const restoreManager = new WalletStorageManager(identityKey, restored) - await restoreManager.makeAvailable() - await restoreManager.syncFromReader(identityKey, client) - const { user: target } = await restored.findOrInsertUser(identityKey) - expect(await restored.countTxLabels({ partial: { userId: target.userId } })).toBe(37) - const noChange = await restoreManager.syncFromReader(identityKey, client) - expect(noChange.inserts).toBe(0) - expect(noChange.updates).toBe(0) - // Exercise real authenticated binary uploads independently of entity merge rules. - const bytes = Array.from({ length: 4096 }, (_, i) => i % 256) - const process = jest.spyOn(remote.activeStorage, 'processSyncChunk').mockResolvedValueOnce({ - done: true, - inserts: 0, - updates: 0, - maxUpdated_at: undefined + test.each([false, true])( + 'backs up unchanged and restores through authenticated HTTP with binary requests %s', + async binaryRequests => { + const remote = await _tu.createSQLiteTestWallet({ + databaseName: `compactCheckpointHttp-${binaryRequests}`, + dropAll: true }) - const authClient = Reflect.get(client, 'authClient') as { fetch: typeof fetch } - const fetchSpy = jest.spyOn(authClient, 'fetch') - await client.processSyncChunk( - { identityKey } as RequestSyncChunkArgs, - { - outputs: [{ created_at: new Date(), updated_at: new Date(), lockingScript: bytes }] - } as never - ) - expect(process.mock.calls[0][1].outputs?.[0].lockingScript).toEqual(bytes) - const body = JSON.parse(String(fetchSpy.mock.calls[0][1]?.body)) - expect(body.params[1].outputs[0].lockingScript.$bsvBinary).toBe('base64') - expect(JSON.stringify(body).length).toBeLessThan(JSON.stringify(bytes).length) - } finally { - await client?.destroy() - await server.close() - await remote.wallet.destroy() - await source.destroy() - await restored.destroy() - await source.dropAllData() - await restored.dropAllData() - } - }, 60000) + const source = await makeStorage() + const restored = await makeStorage() + const server = new StorageServer(remote.activeStorage, { + port: 0, + wallet: remote.wallet, + monetize: false, + logRpcRequests: false, + calculateRequestPrice: async () => 0 + }) + let client: StorageClient | undefined + try { + server.start() + if (!server.server.listening) await once(server.server, 'listening') + const address = server.server.address() + if (address == null || typeof address === 'string') throw new Error('test server did not bind') + client = new StorageClient(remote.wallet, `http://localhost:${address.port}`, { binaryRequests }) + const identityKey = remote.identityKey + const manager = new WalletStorageManager(identityKey, source) + await manager.makeAvailable() + const { user } = await source.findOrInsertUser(identityKey) + for (let i = 0; i < 37; i++) await source.findOrInsertTxLabel(user.userId, `http fixture ${i}`) + const read = source.getSyncChunk.bind(source) + jest.spyOn(source, 'getSyncChunk').mockImplementation(args => read({ ...args, maxItems: 5 })) + const checkpoints = jest.spyOn(client, 'getSyncCheckpoint') + const fullStates = jest.spyOn(client, 'findOrInsertSyncStateAuth') + const committedPages = jest.spyOn(remote.activeStorage, 'processSyncChunk') + const backup = await manager.syncToWriter({ identityKey }, client) + expect(backup.inserts).toBeGreaterThanOrEqual(37) + expect(checkpoints).toHaveBeenCalledTimes(1) + expect(fullStates).not.toHaveBeenCalled() + const unchanged = await manager.syncToWriter({ identityKey }, client) + expect(unchanged).toMatchObject({ inserts: 0, updates: 0 }) + const terminalArgs = committedPages.mock.calls.at(-1)![0] + const terminalReply = (await committedPages.mock.results.at(-1)!.value) as ProcessSyncChunkResult + expect(terminalReply.done).toBe(true) + expect(terminalArgs.offsets.some(entry => entry.offset > 0)).toBe(true) + expect(terminalReply.nextCheckpoint?.offsets.every(entry => entry.offset === 0)).toBe(true) + expect(terminalReply.nextCheckpoint?.since?.getTime()).toBe(new Date(terminalArgs.since!).getTime()) + const upload = client.processSyncChunk.bind(client) + const lostReply = jest.spyOn(client, 'processSyncChunk').mockImplementation(async (args, chunk) => { + const result = await upload(args, chunk) + if (result.done) throw new Error('synthetic lost terminal acknowledgement') + return result + }) + await expect(manager.syncToWriter({ identityKey }, client)).rejects.toThrow( + 'synthetic lost terminal acknowledgement' + ) + lostReply.mockRestore() + await expect(manager.syncToWriter({ identityKey }, client)).resolves.toMatchObject({ inserts: 0, updates: 0 }) + const providerFailure = new WERR_NETWORK_CHAIN('synthetic remote terminal failure') + committedPages.mockResolvedValueOnce({ + error: JSON.parse(WalletError.unknownToJson(providerFailure)), + done: true, + inserts: 999, + updates: 999, + maxUpdated_at: new Date(), + nextCheckpoint: { syncStateId: -1 } as SyncCheckpoint + }) + await expect(manager.syncToWriter({ identityKey }, client)).rejects.toThrow(providerFailure.message) + await expect(manager.syncToWriter({ identityKey }, client)).resolves.toMatchObject({ inserts: 0, updates: 0 }) + const { user: remoteUser } = await remote.activeStorage.findOrInsertUser(identityKey) + expect(await remote.activeStorage.countTxLabels({ partial: { userId: remoteUser.userId } })).toBe(37) + const remoteRead = remote.activeStorage.getSyncChunk.bind(remote.activeStorage) + jest + .spyOn(remote.activeStorage, 'getSyncChunk') + .mockImplementation(args => remoteRead({ ...args, maxItems: 5 })) + const restoreManager = new WalletStorageManager(identityKey, restored) + await restoreManager.makeAvailable() + await restoreManager.syncFromReader(identityKey, client) + const { user: target } = await restored.findOrInsertUser(identityKey) + expect(await restored.countTxLabels({ partial: { userId: target.userId } })).toBe(37) + const noChange = await restoreManager.syncFromReader(identityKey, client) + expect(noChange.inserts).toBe(0) + expect(noChange.updates).toBe(0) + committedPages.mockRestore() + if (binaryRequests) { + // Exercise real authenticated binary uploads independently of entity merge rules. + const bytes = Array.from({ length: 4096 }, (_, i) => i % 256) + const process = jest.spyOn(remote.activeStorage, 'processSyncChunk').mockResolvedValueOnce({ + done: true, + inserts: 0, + updates: 0, + maxUpdated_at: undefined + }) + const authClient = Reflect.get(client, 'authClient') as { fetch: typeof fetch } + const fetchSpy = jest.spyOn(authClient, 'fetch') + await client.processSyncChunk( + { identityKey } as RequestSyncChunkArgs, + { + outputs: [{ created_at: new Date(), updated_at: new Date(), lockingScript: bytes }] + } as never + ) + expect(process.mock.calls[0][1].outputs?.[0].lockingScript).toEqual(bytes) + const body = JSON.parse(String(fetchSpy.mock.calls[0][1]?.body)) + expect(body.params[1].outputs[0].lockingScript.$bsvBinary).toBe('base64') + expect(JSON.stringify(body).length).toBeLessThan(JSON.stringify(bytes).length) + } + } finally { + await client?.destroy() + await server.close() + await remote.wallet.destroy() + await source.destroy() + await restored.destroy() + await source.dropAllData() + await restored.dropAllData() + } + }, + 60000 + ) test('rejects replay of a staged page after its checkpoint has already committed', async () => { const writer = await makeStorage() @@ -268,14 +393,31 @@ describe('compact sync checkpoints', () => { try { const { user } = await writer.findOrInsertUser(identityKey) const checkpoint = await writer.getSyncCheckpoint({ identityKey }, 'source', 'source') - const args = { ...checkpoint, identityKey, fromStorageIdentityKey: 'source', - toStorageIdentityKey: writer.getSettings().storageIdentityKey, maxItems: 1, maxRoughSize: 1024, - requireMatchingCheckpoint: true } + const args = { + ...checkpoint, + identityKey, + fromStorageIdentityKey: 'source', + toStorageIdentityKey: writer.getSettings().storageIdentityKey, + maxItems: 1, + maxRoughSize: 1024, + requireMatchingCheckpoint: true + } const now = new Date() - const chunk = { userIdentityKey: identityKey, fromStorageIdentityKey: 'source', + const chunk = { + userIdentityKey: identityKey, + fromStorageIdentityKey: 'source', toStorageIdentityKey: args.toStorageIdentityKey, - outputTags: [{ outputTagId: 1234, userId: user.userId, created_at: now, updated_at: now, - tag: 'staged replay fixture', isDeleted: false }] } + outputTags: [ + { + outputTagId: 1234, + userId: user.userId, + created_at: now, + updated_at: now, + tag: 'staged replay fixture', + isDeleted: false + } + ] + } await expect(writer.processSyncChunk(args, chunk)).resolves.toMatchObject({ inserts: 1 }) await expect(writer.processSyncChunk(args, chunk)).rejects.toThrow('checkpoint changed') const saved = await writer.getSyncCheckpoint({ identityKey }, 'source', 'source') diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.ts b/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.ts index 8aaf2c7bc..c76dfc5c0 100644 --- a/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.ts +++ b/packages/wallet/wallet-toolbox/src/storage/sync/syncCheckpoint.ts @@ -15,11 +15,16 @@ const entityNames = [ 'provenTxReq' ] +function invalidCheckpoint(): never { + throw new TypeError('Invalid sync checkpoint') +} + /** Validate remote progress and return only the fields permitted to advance a sync. */ -export function validateSyncCheckpoint(value: SyncCheckpoint, previous?: Partial): SyncCheckpoint { - const invalid = (): never => { - throw new TypeError('Invalid sync checkpoint') - } +export function validateSyncCheckpoint( + value: SyncCheckpoint, + previous?: Partial, + completed = false +): SyncCheckpoint { if ( value == null || typeof value !== 'object' || @@ -29,18 +34,29 @@ export function validateSyncCheckpoint(value: SyncCheckpoint, previous?: Partial !Array.isArray(value.offsets) || value.offsets.length !== entityNames.length ) - invalid() + invalidCheckpoint() const offsets = entityNames.map((name, index) => { const entry = value.offsets[index] - if (entry?.name !== name || !Number.isSafeInteger(entry.offset) || entry.offset < 0) invalid() + if (entry?.name !== name || !Number.isSafeInteger(entry.offset) || entry.offset < 0) invalidCheckpoint() return { name, offset: entry.offset } }) let since: Date | undefined if (value.since != null) { - if (!(value.since instanceof Date) && typeof value.since !== 'string') invalid() + if (!(value.since instanceof Date) && typeof value.since !== 'string') invalidCheckpoint() since = new Date(value.since) - if (!Number.isFinite(since.getTime())) invalid() + if (!Number.isFinite(since.getTime())) invalidCheckpoint() } - if (previous?.since != null && (since == null || since < previous.since)) invalid() + if (previous?.since != null && (since == null || since < previous.since)) invalidCheckpoint() + // The legacy terminal page resets every offset, including when equal-time + // rows leave the high-water timestamp unchanged. Other retreats stay invalid. + const completedReset = completed && offsets.every(entry => entry.offset === 0) + if ( + !completedReset && + previous != null && + since?.getTime() === previous.since?.getTime() && + previous.offsets != null && + offsets.some((entry, index) => entry.offset < (previous.offsets?.[index]?.offset ?? 0)) + ) + invalidCheckpoint() return { syncStateId: value.syncStateId, since, offsets } } diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/syncFailure.test.ts b/packages/wallet/wallet-toolbox/src/storage/sync/syncFailure.test.ts new file mode 100644 index 000000000..2e08d28f1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/syncFailure.test.ts @@ -0,0 +1,128 @@ +import 'fake-indexeddb/auto' +import { randomUUID } from 'node:crypto' +import { PrivateKey } from '@bsv/sdk' +import { WalletError } from '../../sdk/WalletError' +import { WalletErrorFromJson } from '../../sdk/WalletErrorFromJson' +import { WERR_INTERNAL, WERR_NETWORK_CHAIN } from '../../sdk/WERR_errors' +import type { Chain } from '../../sdk/types' +import type { ProcessSyncChunkResult } from '../../sdk/WalletStorage.interfaces' +import { StorageIdb } from '../StorageIdb' +import { StorageProvider } from '../StorageProvider' +import { WalletStorageManager } from '../WalletStorageManager' +import { throwSyncResultError } from './syncFailure' + +const stores: StorageIdb[] = [] +const identityKey = PrivateKey.fromRandom().toPublicKey().toString() + +async function makeStorage(chain: Chain = 'test'): Promise { + const storage = new StorageIdb(StorageProvider.createStorageBaseOptions(chain)) + storage.dbName = `sync-failure-${randomUUID()}` + stores.push(storage) + await storage.migrate('sync failure fixture', PrivateKey.fromRandom().toPublicKey().toString()) + await storage.makeAvailable() + return storage +} + +afterEach(async () => { + jest.restoreAllMocks() + for (const storage of stores.splice(0)) { + await storage.destroy() + await storage.dropAllData() + } +}) + +describe.each(['to', 'from'] as const)('sync %s failure contracts', direction => { + function copy(reader: StorageIdb, writer: StorageIdb): Promise { + const manager = new WalletStorageManager(identityKey, direction === 'to' ? reader : writer) + return direction === 'to' + ? manager.syncToWriter({ identityKey }, writer) + : manager.syncFromReader(identityKey, reader) + } + + test.each(['main', 'stn', 'ttn', 'tstn', 'mock'])( + 'rejects test/%s before user registration, checkpoint creation or a chunk read', + async chain => { + const reader = await makeStorage(chain) + const writer = await makeStorage() + const read = jest.spyOn(reader, 'getSyncChunk') + const process = jest.spyOn(writer, 'processSyncChunk') + const register = jest.spyOn(writer, 'findOrInsertUser') + await expect(copy(reader, writer)).rejects.toBeInstanceOf(WERR_NETWORK_CHAIN) + expect(read).not.toHaveBeenCalled() + expect(process).not.toHaveBeenCalled() + expect(register).not.toHaveBeenCalled() + expect(await writer.countUsers({ partial: {} })).toBe(0) + expect(await writer.countSyncStates({ partial: {} })).toBe(0) + } + ) + + test.each([undefined, 'invalid'])('does not guess an absent or unrecognized chain: %s', async chain => { + const reader = await makeStorage() + const writer = await makeStorage() + jest.spyOn(reader, 'makeAvailable').mockResolvedValue({ ...reader.getSettings(), chain } as never) + await expect(copy(reader, writer)).rejects.toBeInstanceOf(WERR_NETWORK_CHAIN) + expect(await writer.countUsers({ partial: {} })).toBe(0) + expect(await writer.countSyncStates({ partial: {} })).toBe(0) + }) + + test.each(['returned', 'serialized', 'thrown'] as const)( + 'preserves a %s failure and resumes only from the durable checkpoint', + async mode => { + const reader = await makeStorage() + const writer = await makeStorage() + const { user } = await reader.findOrInsertUser(identityKey) + await reader.findOrInsertTxLabel(user.userId, 'retained across failure') + const settings = reader.getSettings() + const before = await writer.getSyncCheckpoint({ identityKey }, settings.storageIdentityKey, settings.storageName) + const original = new WERR_NETWORK_CHAIN('custom provider rejected the page') + const error = mode === 'serialized' ? JSON.parse(WalletError.unknownToJson(original)) : original + const process = jest.spyOn(writer, 'processSyncChunk').mockImplementationOnce(async () => { + if (mode === 'thrown') throw error + return { + error, + done: true, + inserts: 999, + updates: 999, + maxUpdated_at: new Date(), + nextCheckpoint: { syncStateId: -1 } + } as ProcessSyncChunkResult + }) + try { + await copy(reader, writer) + throw new Error('sync ignored the provider failure') + } catch (caught) { + expect(caught).toBeInstanceOf(WERR_NETWORK_CHAIN) + expect((caught as Error).message).toBe(original.message) + if (mode !== 'serialized') expect(caught).toBe(original) + } + expect(process).toHaveBeenCalledTimes(1) + expect( + await writer.getSyncCheckpoint({ identityKey }, settings.storageIdentityKey, settings.storageName) + ).toEqual(before) + expect(await writer.countTxLabels({ partial: {} })).toBe(0) + process.mockRestore() + await copy(reader, writer) + expect(await writer.countTxLabels({ partial: {} })).toBe(1) + await copy(reader, writer) + expect(await writer.countTxLabels({ partial: {} })).toBe(1) + } + ) +}) + +test('checks all managed chains before registering a user on any store', async () => { + const active = await makeStorage('main') + const backup = await makeStorage('test') + const manager = new WalletStorageManager(identityKey, active, [backup]) + await expect(manager.makeAvailable()).rejects.toBeInstanceOf(WERR_NETWORK_CHAIN) + expect(await active.countUsers({ partial: {} })).toBe(0) + expect(await backup.countUsers({ partial: {} })).toBe(0) +}) + +test('preserves normal errors, rejects malformed returned failures, and retains JSON identity', () => { + const original = new Error('custom failure') + expect(() => throwSyncResultError({ error: original } as ProcessSyncChunkResult)).toThrow(original) + expect(() => throwSyncResultError({ error: 'malformed' } as never)).toThrow(WERR_INTERNAL) + expect(() => throwSyncResultError({} as ProcessSyncChunkResult)).not.toThrow() + const encoded = WalletError.unknownToJson(new WERR_NETWORK_CHAIN()) + expect(WalletErrorFromJson(JSON.parse(encoded))).toBeInstanceOf(WERR_NETWORK_CHAIN) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/syncFailure.ts b/packages/wallet/wallet-toolbox/src/storage/sync/syncFailure.ts new file mode 100644 index 000000000..26218f276 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/syncFailure.ts @@ -0,0 +1,34 @@ +import { WalletErrorFromJson } from '../../sdk/WalletErrorFromJson' +import { WERR_INVALID_OPERATION, WERR_NETWORK_CHAIN } from '../../sdk/WERR_errors' +import type { RequestSyncChunkArgs, ProcessSyncChunkResult } from '../../sdk/WalletStorage.interfaces' +import type { TableSettings } from '../schema/tables' + +const supportedChains = new Set(['main', 'test', 'stn', 'ttn', 'tstn', 'mock']) + +/** Read-only preflight, before user registration or a destination checkpoint is created. */ +export function assertSyncNetwork(reader: TableSettings, writer: TableSettings): void { + if (!supportedChains.has(reader.chain) || !supportedChains.has(writer.chain) || reader.chain !== writer.chain) { + throw new WERR_NETWORK_CHAIN('Live sync requires matching declared network chains on both storage providers.') + } +} + +/** + * A returned failure has precedence over counters, completion and checkpoint hints. + * Shipped providers normally throw; external providers may use the public error field. + */ +export function throwSyncResultError(result: ProcessSyncChunkResult): void { + const error = result.error + if (error == null) return + if (error instanceof Error) throw error + throw WalletErrorFromJson(error) +} + +/** An unfinished page must advance durable progress, never spin on a provider's empty success. */ +export function assertSyncProgress(before: RequestSyncChunkArgs, after: RequestSyncChunkArgs): void { + const sameSince = before.since?.getTime() === after.since?.getTime() + if (sameSince && after.offsets.every((entry, index) => entry.offset === before.offsets[index]?.offset)) { + throw new WERR_INVALID_OPERATION( + 'Sync provider reported an unfinished page without advancing its durable checkpoint.' + ) + } +} diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/syncSession.test.ts b/packages/wallet/wallet-toolbox/src/storage/sync/syncSession.test.ts new file mode 100644 index 000000000..64d276df6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/syncSession.test.ts @@ -0,0 +1,606 @@ +import 'fake-indexeddb/auto' +import { randomUUID } from 'node:crypto' +import { PrivateKey } from '@bsv/sdk' +import { StorageIdb } from '../StorageIdb' +import { StorageProvider } from '../StorageProvider' +import { WalletStorageManager } from '../WalletStorageManager' +import type { SyncSessionProgress } from './syncSession' + +const stores: StorageIdb[] = [] +const identityKey = PrivateKey.fromRandom().toPublicKey().toString() + +function deferred() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} + +async function makeStorage(dbName?: string): Promise { + const storage = new StorageIdb(StorageProvider.createStorageBaseOptions('test')) + storage.dbName = dbName ?? `sync-session-${randomUUID()}` + stores.push(storage) + await storage.migrate('sync session fixture', PrivateKey.fromRandom().toPublicKey().toString()) + await storage.makeAvailable() + return storage +} + +async function fixture(rows = 25) { + const reader = await makeStorage() + const writer = await makeStorage() + const { user } = await reader.findOrInsertUser(identityKey) + for (let i = 0; i < rows; i++) await reader.findOrInsertTxLabel(user.userId, `source label ${i}`) + const manager = new WalletStorageManager(identityKey, writer) + await manager.makeAvailable() + return { reader, writer, manager } +} + +afterEach(async () => { + jest.restoreAllMocks() + for (const storage of stores) await storage.destroy() + for (const storage of stores.splice(0)) await storage.dropAllData() +}) + +test('foreground reads and writes complete while source I/O waits, with bounded pages and durable progress', async () => { + const { reader, writer, manager } = await fixture() + const entered = deferred() + const release = deferred() + const read = reader.getSyncChunk.bind(reader) + const reads = jest.spyOn(reader, 'getSyncChunk').mockImplementationOnce(async args => { + const chunk = await read(args) + entered.resolve() + await release.promise + return chunk + }) + const events: SyncSessionProgress[] = [] + const sync = manager.syncFromReaderResumable(identityKey, reader, { + maxItems: 3, + maxRoughSize: 4096, + onProgress: event => events.push(event) + }) + await entered.promise + try { + const userId = await manager.getUserId() + await manager.runAsWriter(async () => { + await writer.findOrInsertTxLabel(userId, 'foreground write') + }) + expect(await manager.runAsReader(async () => await writer.countTxLabels({ partial: { userId } }))).toBe(1) + } finally { + release.resolve() + } + const result = await sync + expect(result).toMatchObject({ status: 'completed', mode: 'paged' }) + expect(result.pages).toBeGreaterThan(8) + expect(reads.mock.calls.every(([args]) => args.maxItems <= 3 && args.maxRoughSize <= 4096)).toBe(true) + expect(events.filter(event => event.state === 'committed')).toHaveLength(result.pages) + expect(events.every(event => event.mode === 'paged')).toBe(true) + expect(await writer.countTxLabels({ partial: {} })).toBe(26) + expect((await manager.syncFromReaderResumable(identityKey, reader)).inserts).toBe(0) +}) + +test.each(['earlier timestamp', 'missing timestamp', 'foreign state', 'partial reset'])( + 'rejects a terminal %s reply and resumes from the actual durable checkpoint', + async invalid => { + const { reader, writer, manager } = await fixture(3) + await manager.syncFromReaderResumable(identityKey, reader, { maxItems: 2 }) + const prepare = writer.prepareSyncChunk.bind(writer) + let terminalReplies = 0 + const events: SyncSessionProgress[] = [] + const altered = jest.spyOn(writer, 'prepareSyncChunk').mockImplementation(async (args, chunk) => { + const apply = await prepare(args, chunk) + return async () => { + const result = await apply() + if (result.done) { + terminalReplies++ + const checkpoint = result.nextCheckpoint! + expect(args.since).toBeDefined() + expect(checkpoint.since).toEqual(args.since) + if (invalid === 'earlier timestamp') checkpoint.since = new Date(args.since!.getTime() - 1) + if (invalid === 'missing timestamp') checkpoint.since = undefined + if (invalid === 'foreign state') checkpoint.syncStateId++ + if (invalid === 'partial reset') checkpoint.offsets[0].offset = 1 + } + return result + } + }) + await expect( + manager.syncFromReaderResumable(identityKey, reader, { + maxItems: 2, + onProgress: event => events.push(event) + }) + ).rejects.toThrow('Invalid sync checkpoint') + expect(terminalReplies).toBe(1) + expect(events.some(event => event.state === 'completed')).toBe(false) + altered.mockRestore() + const settings = reader.getSettings() + const durable = await writer.getSyncCheckpoint({ identityKey }, settings.storageIdentityKey, settings.storageName) + expect(durable.since).toBeDefined() + expect(durable.offsets.every(entry => entry.offset === 0)).toBe(true) + await expect(manager.syncFromReaderResumable(identityKey, reader, { maxItems: 2 })).resolves.toMatchObject({ + status: 'completed', + inserts: 0, + updates: 0, + checkpoint: durable + }) + expect(await writer.countTxLabels({ partial: {} })).toBe(3) + } +) + +test('cancellation during a read discards the late page without starting a write', async () => { + const { reader, writer, manager } = await fixture() + const entered = deferred() + const release = deferred() + const read = reader.getSyncChunk.bind(reader) + jest.spyOn(reader, 'getSyncChunk').mockImplementationOnce(async args => { + const chunk = await read(args) + entered.resolve() + await release.promise + return chunk + }) + const process = jest.spyOn(writer, 'prepareSyncChunk') + const controller = new AbortController() + const sync = manager.syncFromReaderResumable(identityKey, reader, { signal: controller.signal }) + await entered.promise + controller.abort() + release.resolve() + expect(await sync).toMatchObject({ status: 'cancelled', pages: 0, inserts: 0 }) + expect(process).not.toHaveBeenCalled() + expect(await writer.countTxLabels({ partial: {} })).toBe(0) +}) + +test('cancellation waits for a committed page acknowledgement and a reopened manager resumes it', async () => { + const { reader, writer, manager } = await fixture() + const entered = deferred() + const release = deferred() + const prepare = writer.prepareSyncChunk.bind(writer) + const spy = jest.spyOn(writer, 'prepareSyncChunk').mockImplementationOnce(async (args, chunk) => { + const commit = await prepare(args, chunk) + return async () => { + const result = await commit() + entered.resolve() + await release.promise + return result + } + }) + const controller = new AbortController() + const states: string[] = [] + let settled = false + const sync = manager + .syncFromReaderResumable(identityKey, reader, { + maxItems: 3, + signal: controller.signal, + onProgress: event => states.push(event.state) + }) + .then(result => { + settled = true + return result + }) + await entered.promise + controller.abort() + await Promise.resolve() + expect(settled).toBe(false) + release.resolve() + const stopped = await sync + expect(stopped).toMatchObject({ status: 'cancelled', pages: 1 }) + expect(stopped.checkpoint?.offsets.some(entry => entry.offset > 0)).toBe(true) + expect(states.slice(-3)).toEqual(['committed', 'cancelling', 'cancelled']) + spy.mockRestore() + await writer.destroy() + const reopened = await makeStorage(writer.dbName) + const resumed = new WalletStorageManager(identityKey, reopened) + expect(await resumed.syncFromReaderResumable(identityKey, reader, { maxItems: 3 })).toMatchObject({ + status: 'completed' + }) + expect(await reopened.countTxLabels({ partial: {} })).toBe(25) + expect(await resumed.syncFromReaderResumable(identityKey, reader)).toMatchObject({ inserts: 0, updates: 0 }) +}) + +test('a lost acknowledgement is not retried; restart loads the committed destination checkpoint', async () => { + const { reader, writer, manager } = await fixture() + const prepare = writer.prepareSyncChunk.bind(writer) + const lost = new Error('lost acknowledgement after commit') + const spy = jest.spyOn(writer, 'prepareSyncChunk').mockImplementationOnce(async (args, chunk) => { + const commit = await prepare(args, chunk) + return async () => { + await commit() + throw lost + } + }) + await expect(manager.syncFromReaderResumable(identityKey, reader, { maxItems: 3 })).rejects.toBe(lost) + expect(spy).toHaveBeenCalledTimes(1) + spy.mockRestore() + const resumed = new WalletStorageManager(identityKey, writer) + await resumed.syncFromReaderResumable(identityKey, reader, { maxItems: 3 }) + expect(await writer.countTxLabels({ partial: {} })).toBe(25) +}) + +test('primary replacement fences a late source reply before it mutates the old writer', async () => { + const { reader, writer } = await fixture() + const replacement = await makeStorage() + const manager = new WalletStorageManager(identityKey, writer, [replacement]) + await manager.makeAvailable() + await manager.setActive(writer.getSettings().storageIdentityKey) + const entered = deferred() + const release = deferred() + const read = reader.getSyncChunk.bind(reader) + jest.spyOn(reader, 'getSyncChunk').mockImplementationOnce(async args => { + const chunk = await read(args) + entered.resolve() + await release.promise + return chunk + }) + const prepare = writer.prepareSyncChunk.bind(writer) + const process = jest.fn() + jest.spyOn(writer, 'prepareSyncChunk').mockImplementation(async (args, chunk) => { + const commit = await prepare(args, chunk) + return async () => { + process() + return await commit() + } + }) + const sync = manager.syncFromReaderResumable(identityKey, reader) + const rejected = expect(sync).rejects.toThrow('destination generation changed') + await entered.promise + try { + await manager.setActive(replacement.getSettings().storageIdentityKey) + } finally { + release.resolve() + } + await rejected + expect(process).not.toHaveBeenCalled() + expect(await writer.countTxLabels({ partial: {} })).toBe(0) + expect(await replacement.countTxLabels({ partial: {} })).toBe(0) +}) + +test('concurrent sessions cannot commit the same checkpoint twice', async () => { + const { reader, writer, manager } = await fixture() + const entered = deferred() + const release = deferred() + const read = reader.getSyncChunk.bind(reader) + let reads = 0 + jest.spyOn(reader, 'getSyncChunk').mockImplementation(async args => { + const chunk = await read(args) + if (++reads <= 2) { + if (reads === 2) entered.resolve() + await release.promise + } + return chunk + }) + const copies = Promise.allSettled([ + manager.syncFromReaderResumable(identityKey, reader, { maxItems: 3 }), + manager.syncFromReaderResumable(identityKey, reader, { maxItems: 3 }) + ]) + await entered.promise + release.resolve() + const outcomes = await copies + expect(outcomes.filter(result => result.status === 'fulfilled')).toHaveLength(1) + const failure = outcomes.find(result => result.status === 'rejected') as PromiseRejectedResult + expect(failure.reason.message).toContain('checkpoint changed') + expect(await writer.countTxLabels({ partial: {} })).toBe(25) +}) + +test('legacy destination capabilities retain exclusive execution and checkpoint fallback', async () => { + const { reader, writer } = await fixture() + jest.spyOn(writer, 'getCapabilities').mockResolvedValue({}) + jest.spyOn(writer, 'getSyncCheckpoint').mockResolvedValue(undefined as never) + const process = writer.processSyncChunk.bind(writer) + jest.spyOn(writer, 'processSyncChunk').mockImplementation(async (args, chunk) => { + const { nextCheckpoint: _, ...result } = await process({ ...args, includeNextCheckpoint: false }, chunk) + return result + }) + const manager = new WalletStorageManager(identityKey, writer) + const entered = deferred() + const release = deferred() + const read = reader.getSyncChunk.bind(reader) + jest.spyOn(reader, 'getSyncChunk').mockImplementationOnce(async args => { + entered.resolve() + await release.promise + return await read(args) + }) + const sync = manager.syncFromReaderResumable(identityKey, reader, { maxItems: 4 }) + await entered.promise + let foreground = false + const queued = manager.runAsReader(async () => { + foreground = true + }) + await Promise.resolve() + expect(foreground).toBe(false) + release.resolve() + expect(await sync).toMatchObject({ status: 'completed', mode: 'exclusive' }) + await queued + expect(foreground).toBe(true) + expect(await writer.countTxLabels({ partial: {} })).toBe(25) +}) + +test('progress observers cannot corrupt checkpoint state and unfinished zero-progress pages terminate', async () => { + const { reader, writer, manager } = await fixture() + await manager.syncFromReaderResumable(identityKey, reader, { + maxItems: 3, + onProgress: event => { + if (event.checkpoint != null) event.checkpoint.offsets[0].offset = 999999 + } + }) + expect(await writer.countTxLabels({ partial: {} })).toBe(25) + const process = jest + .spyOn(writer, 'prepareSyncChunk') + .mockResolvedValue(async () => ({ done: false, inserts: 0, updates: 0, maxUpdated_at: undefined })) + await expect(manager.syncFromReaderResumable(identityKey, reader)).rejects.toThrow('without advancing') + expect(process).toHaveBeenCalledTimes(1) +}) + +test.each([true, false])('manager concurrent reads follow the provider promise (%s)', async concurrent => { + const { writer } = await fixture(0) + const capabilities = writer.getCapabilities.bind(writer) + jest.spyOn(writer, 'getCapabilities').mockImplementation(async () => ({ + ...(await capabilities()), + storageAccess: { version: 1, concurrentReads: concurrent, atomicSyncPages: true } + })) + const manager = new WalletStorageManager(identityKey, writer) + await manager.makeAvailable() + const entered = deferred() + const release = deferred() + const first = manager.runAsReader(async () => { + entered.resolve() + await release.promise + }) + await entered.promise + let secondEntered = false + const second = manager.runAsReader(async () => { + secondEntered = true + }) + await new Promise(resolve => setImmediate(resolve)) + expect(secondEntered).toBe(concurrent) + release.resolve() + await Promise.all([first, second]) +}) + +test('a queued page cancelled behind foreground work never starts its commit', async () => { + const { reader, writer, manager } = await fixture() + const locked = deferred() + const release = deferred() + const controller = new AbortController() + const prepare = writer.prepareSyncChunk.bind(writer) + const applied = jest.fn() + jest.spyOn(writer, 'prepareSyncChunk').mockImplementation(async (args, chunk) => { + const apply = await prepare(args, chunk) + return async () => { + applied() + return await apply() + } + }) + let foreground: Promise | undefined + const sync = manager.syncFromReaderResumable(identityKey, reader, { + signal: controller.signal, + onProgress: event => { + if (event.state === 'committing') + foreground = manager.runAsWriter(async () => { + locked.resolve() + await release.promise + }) + } + }) + await locked.promise + controller.abort() + release.resolve() + await foreground + expect(await sync).toMatchObject({ status: 'cancelled', pages: 0 }) + expect(applied).not.toHaveBeenCalled() + expect(await writer.countTxLabels({ partial: {} })).toBe(0) +}) + +test.each(['fromStorageIdentityKey', 'toStorageIdentityKey', 'userIdentityKey'] as const)( + 'rejects a page with the wrong %s before preparation', + async key => { + const { reader, writer, manager } = await fixture() + const get = reader.getSyncChunk.bind(reader) + jest + .spyOn(reader, 'getSyncChunk') + .mockImplementation(async args => ({ ...(await get(args)), [key]: 'wrong identity' })) + const prepare = jest.spyOn(writer, 'prepareSyncChunk') + await expect(manager.syncFromReaderResumable(identityKey, reader)).rejects.toThrow('bound to this sync') + expect(prepare).not.toHaveBeenCalled() + expect(await writer.countTxLabels({ partial: {} })).toBe(0) + } +) + +test.each([{ maxItems: 0 }, { maxItems: 1001 }, { maxRoughSize: Infinity }, { maxRoughSize: 1.5 }])( + 'rejects invalid bounds before fetching a page: %o', + async options => { + const { reader, writer, manager } = await fixture() + const read = jest.spyOn(reader, 'getSyncChunk') + await expect(manager.syncFromReaderResumable(identityKey, reader, options)).rejects.toThrow('an integer from') + expect(read).not.toHaveBeenCalled() + expect(await writer.countSyncStates({ partial: {} })).toBe(0) + } +) + +test('IndexedDB indexed paging retains filtered offset and timestamp semantics', async () => { + const { reader } = await fixture(100) + const { user } = await reader.findOrInsertUser(identityKey) + const all = await reader.findTxLabels({ partial: { userId: user.userId } }) + const page = await reader.findTxLabels({ partial: { userId: user.userId }, paged: { offset: 80, limit: 5 } }) + expect(page).toEqual(all.slice(80, 85)) + await reader.updateTxLabel(all[85].txLabelId, { isDeleted: true, updated_at: new Date('2099-01-01') }) + expect( + await reader.findTxLabels({ partial: { userId: user.userId, isDeleted: true }, paged: { offset: 1, limit: 5 } }) + ).toEqual([]) + expect( + await reader.findTxLabels({ + partial: { userId: user.userId }, + since: new Date('2098-01-01'), + paged: { offset: 1, limit: 5 } + }) + ).toEqual([]) + expect( + await reader.findTxLabels({ + partial: { userId: user.userId }, + since: new Date('2098-01-01'), + paged: { offset: 0, limit: 5 } + }) + ).toMatchObject([{ txLabelId: all[85].txLabelId }]) +}) + +test.each([undefined, 10000000])( + 'uses a bounded new default without narrowing the explicit legacy ceiling (%s)', + async maximum => { + const { reader, manager } = await fixture(3) + const calls = jest.spyOn(reader, 'getSyncChunk') + await manager.syncFromReaderResumable(identityKey, reader, { maxRoughSize: maximum }) + expect(calls.mock.calls.length).toBeGreaterThan(0) + expect(calls.mock.calls.every(([args]) => args.maxRoughSize === (maximum ?? 262144))).toBe(true) + } +) + +test.each([false, true])( + 'queued primary reselection waits for the earlier switch, including failure=%s', + async failEarlier => { + const first = await makeStorage(), + second = await makeStorage() + const manager = new WalletStorageManager(identityKey, first, [second]) + await manager.makeAvailable() + const firstKey = first.getSettings().storageIdentityKey, + secondKey = second.getSettings().storageIdentityKey + await manager.setActive(firstKey) + const entered = deferred(), + release = deferred(), + failure = new Error('synthetic primary switch failure') + const update = first.setActive.bind(first) + jest.spyOn(first, 'setActive').mockImplementationOnce(async (...args) => { + entered.resolve() + await release.promise + if (failEarlier) throw failure + return await update(...args) + }) + const switching = manager.setActive(secondKey) + const checkedSwitch = failEarlier ? expect(switching).rejects.toBe(failure) : switching + await entered.promise + let settled = false + const reselecting = manager.setActive(firstKey).then(log => { + settled = true + return log + }) + try { + await new Promise(resolve => setImmediate(resolve)) + expect(settled).toBe(false) + } finally { + release.resolve() + await checkedSwitch + await reselecting + } + expect(manager.getActiveStore()).toBe(firstKey) + expect(manager.isActiveEnabled).toBe(true) + expect((await first.findUserByIdentityKey(identityKey))!.activeStorage).toBe(firstKey) + expect((await second.findUserByIdentityKey(identityKey))!.activeStorage).toBe(firstKey) + } +) + +test('a failed primary-switch prelude leaves an in-flight sync on the same primary usable', async () => { + const { reader, writer } = await fixture(3), + replacement = await makeStorage() + const manager = new WalletStorageManager(identityKey, writer, [replacement]) + await manager.makeAvailable() + const current = writer.getSettings().storageIdentityKey + await manager.setActive(current) + const entered = deferred(), + release = deferred(), + failure = new Error('synthetic primary progress observer failure') + const read = reader.getSyncChunk.bind(reader) + jest.spyOn(reader, 'getSyncChunk').mockImplementationOnce(async args => { + const chunk = await read(args) + entered.resolve() + await release.promise + return chunk + }) + const copying = manager.syncFromReaderResumable(identityKey, reader, { maxItems: 2 }) + const checkedCopy = expect(copying).resolves.toMatchObject({ status: 'completed', mode: 'paged' }) + await entered.promise + try { + await expect( + manager.setActive(replacement.getSettings().storageIdentityKey, message => { + if (message === '\n') throw failure + return message + }) + ).rejects.toBe(failure) + expect(manager.getActiveStore()).toBe(current) + } finally { + release.resolve() + await checkedCopy + } + expect(await writer.countTxLabels({ partial: {} })).toBe(3) +}) + +test.each(['', 'unregistered'])( + 'an invalid primary %s initializes safely and refuses before provider mutation', + async key => { + const first = await makeStorage() + const manager = new WalletStorageManager(identityKey, first) + const update = jest.spyOn(first, 'setActive') + const format = jest.fn((message: string) => message) + expect(manager.isAvailable()).toBe(false) + await expect(manager.setActive(key, format)).rejects.toMatchObject({ code: 'WERR_INVALID_PARAMETER' }) + expect(manager.getActiveStore()).toBe(first.getSettings().storageIdentityKey) + expect(manager.isActiveEnabled).toBe(true) + expect(update).not.toHaveBeenCalled() + expect(format).not.toHaveBeenCalled() + expect(await manager.setActive(first.getSettings().storageIdentityKey)).toContain(' unchanged\n') + } +) + +test('a partial primary propagation failure reloads persisted selection before authorizing or retrying', async () => { + const first = await makeStorage(), + second = await makeStorage() + const manager = new WalletStorageManager(identityKey, first, [second]) + await manager.makeAvailable() + const firstKey = first.getSettings().storageIdentityKey, + secondKey = second.getSettings().storageIdentityKey + await manager.setActive(firstKey) + const failure = new Error('synthetic partial propagation failure') + jest.spyOn(second, 'processSyncChunk').mockRejectedValueOnce(failure) + await expect(manager.setActive(secondKey)).rejects.toBe(failure) + expect((await first.findUserByIdentityKey(identityKey))!.activeStorage).toBe(secondKey) + expect((await second.findUserByIdentityKey(identityKey))!.activeStorage).toBe(firstKey) + await expect(manager.getAuth(true)).rejects.toMatchObject({ code: 'WERR_NOT_ACTIVE' }) + expect(manager.isActiveEnabled).toBe(false) + await manager.setActive(firstKey) + expect(manager.isActiveEnabled).toBe(true) + expect((await first.findUserByIdentityKey(identityKey))!.activeStorage).toBe(firstKey) + expect((await second.findUserByIdentityKey(identityKey))!.activeStorage).toBe(firstKey) +}) + +test('a queued request releases ownership after a failed primary recovery reload', async () => { + const first = await makeStorage(), + second = await makeStorage() + const manager = new WalletStorageManager(identityKey, first, [second]) + await manager.makeAvailable() + const firstKey = first.getSettings().storageIdentityKey, + secondKey = second.getSettings().storageIdentityKey + await manager.setActive(firstKey) + const entered = deferred(), + release = deferred() + const failure = new Error('synthetic partial propagation failure'), + reloadFailure = new Error('synthetic recovery reload failure') + jest.spyOn(second, 'processSyncChunk').mockImplementationOnce(async () => { + entered.resolve() + await release.promise + throw failure + }) + const switching = expect(manager.setActive(secondKey)).rejects.toBe(failure) + await entered.promise + jest.spyOn(first, 'findOrInsertUser').mockRejectedValueOnce(reloadFailure) + const operation = jest.fn(async () => undefined) + const queued = expect(manager.runAsWriter(operation)).rejects.toBe(reloadFailure) + try { + await new Promise(resolve => setImmediate(resolve)) + expect(operation).not.toHaveBeenCalled() + } finally { + release.resolve() + await switching + await queued + } + expect(operation).not.toHaveBeenCalled() + expect(await manager.runAsReader(async provider => provider.getSettings().storageIdentityKey)).toBe(firstKey) + await expect(manager.getAuth(true)).rejects.toMatchObject({ code: 'WERR_NOT_ACTIVE' }) + await manager.setActive(firstKey) + expect((await manager.getAuth(true)).isActive).toBe(true) +}) diff --git a/packages/wallet/wallet-toolbox/src/storage/sync/syncSession.ts b/packages/wallet/wallet-toolbox/src/storage/sync/syncSession.ts new file mode 100644 index 000000000..da98da1ab --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/storage/sync/syncSession.ts @@ -0,0 +1,195 @@ +import type { SnapshotSyncCheckpoint } from '../snapshot/SnapshotSync' +import type { + ProcessSyncChunkResult, + RequestSyncChunkArgs, + SyncCheckpoint, + SyncChunk, + WalletStorageSync, + WalletStorageSyncReader +} from '../../sdk/WalletStorage.interfaces' +import { WERR_INVALID_PARAMETER } from '../../sdk/WERR_errors' +import { SyncPageBudget } from './SyncPageBudget' +import { validateSyncCheckpoint } from './syncCheckpoint' +import { assertSyncProgress, throwSyncResultError } from './syncFailure' + +export interface SyncSessionOptions { + /** Cancellation waits for an in-flight commit acknowledgement; it never rolls back an acknowledged page. */ + signal?: AbortSignal + /** Local snapshot retention, default five minutes and at most one hour. Expiry requires a new view. */ + snapshotLifetimeMs?: number + /** Additional page ceilings; defaults are 1,000 rows and 262,144 rough encoded bytes. */ + maxItems?: number + maxRoughSize?: number + /** Called outside page ownership in paged mode. Checkpoints are independent copies. */ + onProgress?: (progress: SyncSessionProgress) => void +} + +export interface SyncSessionProgress { + state: 'reading' | 'preparing' | 'committing' | 'committed' | 'cancelling' | 'cancelled' | 'completed' + mode: 'paged' | 'exclusive' + pages: number + inserts: number + updates: number + /** Present for negotiated local snapshot sync; legacy offsets remain separate. */ + snapshotCheckpoint?: SnapshotSyncCheckpoint + checkpoint?: SyncCheckpoint + readMs?: number + prepareMs?: number + queueMs?: number + commitMs?: number +} + +export interface SyncSessionResult { + status: 'completed' | 'cancelled' + mode: 'paged' | 'exclusive' + pages: number + inserts: number + updates: number + /** Present for negotiated local snapshot sync; legacy offsets remain separate. */ + snapshotCheckpoint?: SnapshotSyncCheckpoint + checkpoint?: SyncCheckpoint +} + +interface PullSession { + reader: WalletStorageSyncReader + writer: WalletStorageSync + mode: 'paged' | 'exclusive' + /** Pull preserves its destination selection; push forwards the source row unchanged. */ + activeStorage?: string + atomicCheckpoint: boolean + loadRequest: () => Promise + prepare?: (args: RequestSyncChunkArgs, chunk: SyncChunk) => Promise<() => Promise> + /** Must check destination/generation inside ownership immediately before any write. */ + commit: (operation: () => Promise) => Promise +} + +function boundedOption(value: number | undefined, name: string, maximum: number): number { + if (value === undefined) return maximum + if (!Number.isSafeInteger(value) || value < 1 || value > maximum) { + throw new WERR_INVALID_PARAMETER(name, `an integer from 1 to ${maximum}`) + } + return value +} + +function requestCheckpoint(args: RequestSyncChunkArgs): SyncCheckpoint { + return validateSyncCheckpoint({ syncStateId: args.syncStateId ?? 0, since: args.since, offsets: args.offsets }) +} + +type ProgressNotifier = (state: SyncSessionProgress['state'], timing?: Partial) => void + +async function readAndPreparePage( + session: PullSession, + pageArgs: RequestSyncChunkArgs, + cancelled: () => boolean, + notify: ProgressNotifier +): Promise< + { chunk: SyncChunk; apply: () => Promise; readMs: number; prepareMs: number } | undefined +> { + notify('reading') + if (cancelled()) return undefined + const readAt = Date.now() + const chunk = await session.reader.getSyncChunk(pageArgs) + const readMs = Date.now() - readAt + if (cancelled()) return undefined + if ( + chunk.fromStorageIdentityKey !== pageArgs.fromStorageIdentityKey || + chunk.toStorageIdentityKey !== pageArgs.toStorageIdentityKey || + chunk.userIdentityKey !== pageArgs.identityKey + ) { + throw new WERR_INVALID_PARAMETER('chunk', 'bound to this sync source, destination and wallet identity') + } + if (chunk.user != null && session.activeStorage !== undefined) chunk.user.activeStorage = session.activeStorage + notify('preparing', { readMs }) + if (cancelled()) return undefined + const prepareAt = Date.now() + const apply = + session.prepare == null + ? () => session.writer.processSyncChunk(pageArgs, chunk) + : await session.prepare(pageArgs, chunk) + const prepareMs = Date.now() - prepareAt + if (cancelled()) return undefined + return { chunk, apply, readMs, prepareMs } +} + +async function committedCheckpoint( + session: PullSession, + args: RequestSyncChunkArgs, + reply: ProcessSyncChunkResult +): Promise { + if (reply.nextCheckpoint == null) return requestCheckpoint(await session.loadRequest()) + return validateSyncCheckpoint(reply.nextCheckpoint, args, reply.done === true) +} + +/** One page in flight; resume always starts with the destination's durable checkpoint. */ +export async function runPullSession(session: PullSession, options: SyncSessionOptions): Promise { + const maxItems = boundedOption(options.maxItems, 'maxItems', 1000) + const maxRoughSize = boundedOption(options.maxRoughSize ?? 262144, 'maxRoughSize', 10000000) + const { signal, onProgress } = options + const result: SyncSessionResult = { status: 'completed', mode: session.mode, pages: 0, inserts: 0, updates: 0 } + const notify = (state: SyncSessionProgress['state'], timing: Partial = {}): void => { + onProgress?.({ + ...result, + ...timing, + state, + checkpoint: result.checkpoint == null ? undefined : validateSyncCheckpoint(result.checkpoint) + }) + } + const cancelled = (): boolean => { + if (signal?.aborted !== true) return false + result.status = 'cancelled' + notify('cancelling') + notify('cancelled') + return true + } + if (cancelled()) return result + let args = await session.commit(session.loadRequest) + result.checkpoint = requestCheckpoint(args) + args = { + ...args, + maxItems: Math.min(args.maxItems, maxItems), + maxRoughSize: Math.min(args.maxRoughSize, maxRoughSize) + } + const budget = new SyncPageBudget() + for (;;) { + if (cancelled()) return result + const pageArgs = { + ...budget.apply(args), + includeNextCheckpoint: true, + requireMatchingCheckpoint: session.atomicCheckpoint + } + const prepared = await readAndPreparePage(session, pageArgs, cancelled, notify) + if (prepared == null) return result + const { chunk, apply, readMs, prepareMs } = prepared + notify('committing', { readMs }) + const queuedAt = Date.now() + let commitAt = queuedAt + const committed = await session.commit(async () => { + // A cancellation requested while queued must not start a destination write. + if (signal?.aborted === true) return undefined + commitAt = Date.now() + const reply = await apply() + throwSyncResultError(reply) + const checkpoint = await committedCheckpoint(session, args, reply) + return { reply, checkpoint } + }) + if (committed == null) { + cancelled() + return result + } + const commitMs = Date.now() - commitAt + const { reply, checkpoint } = committed + if (!reply.done) assertSyncProgress(args, { ...args, ...checkpoint }) + budget.committed(chunk, readMs + prepareMs + commitMs, readMs + prepareMs) + result.pages++ + result.inserts += reply.inserts + result.updates += reply.updates + result.checkpoint = checkpoint + notify('committed', { readMs, prepareMs, queueMs: commitAt - queuedAt, commitMs }) + if (cancelled()) return result + if (reply.done) { + notify('completed') + return result + } + args = { ...args, ...checkpoint } + } +} diff --git a/packages/wallet/wallet-toolbox/src/utility/__tests__/runInSeries.test.ts b/packages/wallet/wallet-toolbox/src/utility/__tests__/runInSeries.test.ts new file mode 100644 index 000000000..34efc79f5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/utility/__tests__/runInSeries.test.ts @@ -0,0 +1,49 @@ +import { runInSeries } from '../runInSeries' + +test('pulls lazily and starts only one operation at a time', async () => { + let release!: () => void + const gate = new Promise(resolve => { + release = resolve + }) + const started: number[] = [] + const completed: number[] = [] + const pending = runInSeries([1, 2, 3], async value => { + started.push(value) + if (value === 1) await gate + expect(completed).toEqual(started.slice(0, -1)) + completed.push(value) + return value * 2 + }) + await Promise.resolve() + expect(started).toEqual([1]) + expect(completed).toEqual([]) + release() + await expect(pending).resolves.toBe(6) + expect(completed).toEqual([1, 2, 3]) +}) + +test.each(['throw', 'reject'] as const)('stops and closes its input on an operation %s', async mode => { + let closed = false + function* values() { + try { + yield* [1, 2, 3] + } finally { + closed = true + } + } + const failure = new Error('operation failed') + const work = jest.fn((value: number) => { + if (value !== 2) return Promise.resolve(value) + if (mode === 'throw') throw failure + return Promise.reject(failure) + }) + await expect(runInSeries(values(), work)).rejects.toBe(failure) + expect(work.mock.calls).toEqual([[1], [2]]) + expect(closed).toBe(true) +}) + +test('does not call work for an empty source', async () => { + const work = jest.fn(() => Promise.resolve(1)) + await expect(runInSeries([], work)).resolves.toBeUndefined() + expect(work).not.toHaveBeenCalled() +}) diff --git a/packages/wallet/wallet-toolbox/src/utility/runInSeries.ts b/packages/wallet/wallet-toolbox/src/utility/runInSeries.ts new file mode 100644 index 000000000..b9812dc88 --- /dev/null +++ b/packages/wallet/wallet-toolbox/src/utility/runInSeries.ts @@ -0,0 +1,11 @@ +/** Start each operation only after its predecessor settles; return the last result. */ +export async function runInSeries(values: Iterable, work: (value: T) => Promise): Promise { + async function* results() { + for (const value of values) yield work(value) + } + let last: R | undefined + for await (last of results()) { + // Pull lazily: the async generator awaits each operation before accepting the next. + } + return last +} diff --git a/packages/wallet/wallet-toolbox/test/consumer/hashWasm.cts b/packages/wallet/wallet-toolbox/test/consumer/hashWasm.cts new file mode 100644 index 000000000..820ffd7e1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/consumer/hashWasm.cts @@ -0,0 +1,25 @@ +// Check the advertised CommonJS leaf from both CommonJS and ESM consumers. +import hashWasm = require('@bsv/wallet-toolbox/out/src/utility/hashWasm') + +type IsAny = 0 extends 1 & T ? true : false +const hasImplicitAny: IsAny = false +const original: typeof import('hash-wasm').argon2id = hashWasm.argon2id +const binary: Promise = hashWasm.argon2id({ + password: new Uint8Array([1]), + salt: new Uint8Array(16), + iterations: 1, + parallelism: 1, + memorySize: 8, + hashLength: 32, + outputType: 'binary' +}) +const encoded: Promise = hashWasm.argon2id({ + password: 'synthetic', + salt: 'synthetic-salt', + iterations: 1, + parallelism: 1, + memorySize: 8, + hashLength: 32, + outputType: 'encoded' +}) +export { hasImplicitAny, original, binary, encoded } diff --git a/packages/wallet/wallet-toolbox/test/consumer/hashWasm.mts b/packages/wallet/wallet-toolbox/test/consumer/hashWasm.mts new file mode 100644 index 000000000..820ffd7e1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/consumer/hashWasm.mts @@ -0,0 +1,25 @@ +// Check the advertised CommonJS leaf from both CommonJS and ESM consumers. +import hashWasm = require('@bsv/wallet-toolbox/out/src/utility/hashWasm') + +type IsAny = 0 extends 1 & T ? true : false +const hasImplicitAny: IsAny = false +const original: typeof import('hash-wasm').argon2id = hashWasm.argon2id +const binary: Promise = hashWasm.argon2id({ + password: new Uint8Array([1]), + salt: new Uint8Array(16), + iterations: 1, + parallelism: 1, + memorySize: 8, + hashLength: 32, + outputType: 'binary' +}) +const encoded: Promise = hashWasm.argon2id({ + password: 'synthetic', + salt: 'synthetic-salt', + iterations: 1, + parallelism: 1, + memorySize: 8, + hashLength: 32, + outputType: 'encoded' +}) +export { hasImplicitAny, original, binary, encoded } diff --git a/packages/wallet/wallet-toolbox/test/consumer/tsconfig.json b/packages/wallet/wallet-toolbox/test/consumer/tsconfig.json new file mode 100644 index 000000000..fd19be7a4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/consumer/tsconfig.json @@ -0,0 +1,13 @@ +{ + "extends": "../../../../../config/typescript/test.json", + "compilerOptions": { + "target": "ES2022", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "strict": true, + "skipLibCheck": false, + "noEmit": true, + "types": ["node"] + }, + "files": ["hashWasm.cts", "hashWasm.mts"] +} diff --git a/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-ddl-fixture.json b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-ddl-fixture.json new file mode 100644 index 000000000..31d5c1a04 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-ddl-fixture.json @@ -0,0 +1,250 @@ +{ + "epoch": "338b08b7-6fd5-4369-a892-06aa53896066", + "ddl": [ + { + "sql": "CREATE TABLE snapshot_journal_generation(id INTEGER NOT NULL PRIMARY KEY,version INTEGER NOT NULL,epoch VARCHAR(36) NOT NULL,source VARCHAR(64) NOT NULL,plan VARCHAR(64) NOT NULL,ceiling VARCHAR(19) NOT NULL,nextObject INTEGER NOT NULL,complete BOOLEAN NOT NULL) ENGINE=InnoDB DEFAULT CHARACTER SET ascii COLLATE ascii_bin ROW_FORMAT=DYNAMIC SELECT 1 id,1 version,? epoch,? source,? plan,? ceiling,0 nextObject,0 complete", + "bindings": [ + "338b08b7-6fd5-4369-a892-06aa53896066", + "166b30f8f3e2c27b0bdc36a22b1aa86514cd245a3ad0ce84f69c215890a3b2ee", + "8d32f91f5cf64eb06b842835b18ebd86121a90388efa3e219cbddb71c0320309", + "9223372036854775807" + ] + }, + { + "sql": "CREATE TABLE snapshot_journal_clock(id INTEGER NOT NULL PRIMARY KEY,ceiling BIGINT UNSIGNED NOT NULL,CHECK(id=1)) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066' SELECT 1 id,? ceiling", + "bindings": ["9223372036854775807"] + }, + { + "sql": "CREATE TABLE snapshot_journal_events(revision BIGINT UNSIGNED NOT NULL AUTO_INCREMENT PRIMARY KEY) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066'", + "bindings": [] + }, + { + "sql": "CREATE TABLE snapshot_journal_invalid(id INTEGER NOT NULL PRIMARY KEY,reason VARCHAR(32) NOT NULL,CHECK(id=1),CHECK(reason IN ('capacity-exhausted','revision-exhausted','key-out-of-range'))) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066'", + "bindings": [] + }, + { + "sql": "CREATE TABLE snapshot_journal_physical(tableId INT NOT NULL,id1 BIGINT UNSIGNED NOT NULL,id2 BIGINT UNSIGNED NOT NULL,exactText VARBINARY(400) NOT NULL,revision BIGINT UNSIGNED NOT NULL,generation BIGINT UNSIGNED NOT NULL,present BOOLEAN NOT NULL,PRIMARY KEY(tableId,id1,id2,exactText),KEY snapshot_journal_physical_page(tableId,revision,id1,id2,exactText)) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066'", + "bindings": [] + }, + { + "sql": "CREATE TABLE snapshot_journal_scope(tableId INT NOT NULL,userId BIGINT UNSIGNED NOT NULL,id1 BIGINT UNSIGNED NOT NULL,id2 BIGINT UNSIGNED NOT NULL,exactText VARBINARY(400) NOT NULL,revision BIGINT UNSIGNED NOT NULL,present BOOLEAN NOT NULL,PRIMARY KEY(tableId,userId,id1,id2,exactText),KEY snapshot_journal_scope_page(userId,tableId,revision,id1,id2,exactText)) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066'", + "bindings": [] + }, + { + "sql": "CREATE TABLE snapshot_journal_bootstrap(id INTEGER NOT NULL PRIMARY KEY,stream INTEGER NOT NULL,`cursor` TEXT,rowLimit INTEGER,rowsUsed INTEGER NOT NULL,CHECK(id=1),CHECK(stream BETWEEN 0 AND 17),CHECK(rowLimit IS NULL OR rowLimit BETWEEN 0 AND 2147483647),CHECK(rowsUsed BETWEEN 0 AND 2147483647),CHECK(rowLimit IS NULL OR rowsUsed<=rowLimit)) ENGINE=InnoDB DEFAULT CHARACTER SET utf8mb4 COLLATE utf8mb4_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066' SELECT 1 id,0 stream,NULL `cursor`,NULL rowLimit,0 rowsUsed", + "bindings": [] + }, + { + "sql": "CREATE TABLE snapshot_journal_retention(id INTEGER NOT NULL PRIMARY KEY,floor VARCHAR(19) NOT NULL,receiptLimit INTEGER NOT NULL,receiptLifetimeMs BIGINT NOT NULL,CHECK(id=1),CHECK(receiptLimit BETWEEN 1 AND 128),CHECK(receiptLifetimeMs BETWEEN 1 AND 2592000000)) ENGINE=InnoDB DEFAULT CHARACTER SET ascii COLLATE ascii_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066' SELECT 1 id,'0' floor,? receiptLimit,? receiptLifetimeMs", + "bindings": [128, 2592000000] + }, + { + "sql": "CREATE TABLE snapshot_journal_receipts(requestId VARCHAR(64) NOT NULL PRIMARY KEY,binding VARCHAR(64) NOT NULL,highWater VARCHAR(19) NOT NULL,floor VARCHAR(19) NOT NULL,expiresAt BIGINT NOT NULL,CHECK(expiresAt BETWEEN 1 AND 9007199254740991),KEY snapshot_journal_receipts_expiry(expiresAt,requestId)) ENGINE=InnoDB DEFAULT CHARACTER SET ascii COLLATE ascii_bin ROW_FORMAT=DYNAMIC COMMENT='snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066'", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_0_INSERT AFTER INSERT ON `snapshot_profile_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotRowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId,NEW.snapshotUserId,NEW.snapshotRowId,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_0_DELETE AFTER DELETE ON `snapshot_profile_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.snapshotRowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT OLD.snapshotTableId,OLD.snapshotUserId,OLD.snapshotRowId,0,CAST('' AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_0_UPDATE AFTER UPDATE ON `snapshot_profile_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (OLD.snapshotTableId <=> NEW.snapshotTableId) OR NOT (OLD.snapshotUserId <=> NEW.snapshotUserId) OR NOT (OLD.snapshotRowId <=> NEW.snapshotRowId) OR NOT (0 <=> 0) OR NOT (CAST('' AS BINARY) <=> CAST('' AS BINARY)) OR NOT (1 <=> 1) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotRowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId,NEW.snapshotUserId,NEW.snapshotRowId,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_1_INSERT AFTER INSERT ON `snapshot_relation_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotLeftId BETWEEN 1 AND 9007199254740991) AND (NEW.snapshotRightId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId+10,NEW.snapshotUserId,NEW.snapshotLeftId,NEW.snapshotRightId,CAST('' AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_1_DELETE AFTER DELETE ON `snapshot_relation_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.snapshotLeftId BETWEEN 1 AND 9007199254740991) AND (OLD.snapshotRightId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT OLD.snapshotTableId+10,OLD.snapshotUserId,OLD.snapshotLeftId,OLD.snapshotRightId,CAST('' AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_1_UPDATE AFTER UPDATE ON `snapshot_relation_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (OLD.snapshotTableId+10 <=> NEW.snapshotTableId+10) OR NOT (OLD.snapshotUserId <=> NEW.snapshotUserId) OR NOT (OLD.snapshotLeftId <=> NEW.snapshotLeftId) OR NOT (OLD.snapshotRightId <=> NEW.snapshotRightId) OR NOT (CAST('' AS BINARY) <=> CAST('' AS BINARY)) OR NOT ((OLD.snapshotMembership<>0) <=> (NEW.snapshotMembership<>0)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotLeftId BETWEEN 1 AND 9007199254740991) AND (NEW.snapshotRightId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId+10,NEW.snapshotUserId,NEW.snapshotLeftId,NEW.snapshotRightId,CAST('' AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_2_INSERT AFTER INSERT ON `snapshot_certificate_field_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotCertificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.snapshotFieldName AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,NEW.snapshotUserId,NEW.snapshotCertificateId,0,CAST(NEW.snapshotFieldName AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_2_DELETE AFTER DELETE ON `snapshot_certificate_field_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.snapshotCertificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(OLD.snapshotFieldName AS BINARY))<=400) AND (OLD.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,OLD.snapshotUserId,OLD.snapshotCertificateId,0,CAST(OLD.snapshotFieldName AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_2_UPDATE AFTER UPDATE ON `snapshot_certificate_field_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (12 <=> 12) OR NOT (OLD.snapshotUserId <=> NEW.snapshotUserId) OR NOT (OLD.snapshotCertificateId <=> NEW.snapshotCertificateId) OR NOT (0 <=> 0) OR NOT (CAST(OLD.snapshotFieldName AS BINARY) <=> CAST(NEW.snapshotFieldName AS BINARY)) OR NOT ((OLD.snapshotMembership<>0) <=> (NEW.snapshotMembership<>0)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotCertificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.snapshotFieldName AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,NEW.snapshotUserId,NEW.snapshotCertificateId,0,CAST(NEW.snapshotFieldName AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_3_INSERT AFTER INSERT ON `snapshot_global_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.rowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT CASE NEW.tableId WHEN 0 THEN 9 ELSE 8 END,NEW.userId,NEW.rowId,0,CAST('' AS BINARY),journalRevision,NEW.present ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_3_DELETE AFTER DELETE ON `snapshot_global_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.rowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT CASE OLD.tableId WHEN 0 THEN 9 ELSE 8 END,OLD.userId,OLD.rowId,0,CAST('' AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_scope_3_UPDATE AFTER UPDATE ON `snapshot_global_keys` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CASE OLD.tableId WHEN 0 THEN 9 ELSE 8 END <=> CASE NEW.tableId WHEN 0 THEN 9 ELSE 8 END) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.rowId <=> NEW.rowId) OR NOT (0 <=> 0) OR NOT (CAST('' AS BINARY) <=> CAST('' AS BINARY)) OR NOT (OLD.present <=> NEW.present) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.rowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT CASE NEW.tableId WHEN 0 THEN 9 ELSE 8 END,NEW.userId,NEW.rowId,0,CAST('' AS BINARY),journalRevision,NEW.present ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_0_INSERT AFTER INSERT ON `transactions` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 0,NEW.userId,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_0_UPDATE AFTER UPDATE ON `transactions` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`provenTxId` AS BINARY) <=> CAST(NEW.`provenTxId` AS BINARY)) OR NOT (CAST(OLD.`status` AS BINARY) <=> CAST(NEW.`status` AS BINARY)) OR NOT (CAST(OLD.`reference` AS BINARY) <=> CAST(NEW.`reference` AS BINARY)) OR NOT (CAST(OLD.`isOutgoing` AS BINARY) <=> CAST(NEW.`isOutgoing` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`version` AS BINARY) <=> CAST(NEW.`version` AS BINARY)) OR NOT (CAST(OLD.`lockTime` AS BINARY) <=> CAST(NEW.`lockTime` AS BINARY)) OR NOT (CAST(OLD.`description` AS BINARY) <=> CAST(NEW.`description` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`inputBEEF` AS BINARY) <=> CAST(NEW.`inputBEEF` AS BINARY)) OR NOT (CAST(OLD.`rawTx` AS BINARY) <=> CAST(NEW.`rawTx` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryMode` AS BINARY) <=> CAST(NEW.`noSendExpiryMode` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryValue` AS BINARY) <=> CAST(NEW.`noSendExpiryValue` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryDeadline` AS BINARY) <=> CAST(NEW.`noSendExpiryDeadline` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryState` AS BINARY) <=> CAST(NEW.`noSendExpiryState` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryAnchorTxid` AS BINARY) <=> CAST(NEW.`noSendExpiryAnchorTxid` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryAnchorVout` AS BINARY) <=> CAST(NEW.`noSendExpiryAnchorVout` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReleasedAt` AS BINARY) <=> CAST(NEW.`noSendExpiryReleasedAt` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryObservedAt` AS BINARY) <=> CAST(NEW.`noSendExpiryObservedAt` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimTxid` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimTxid` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimRawTx` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimRawTx` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimDerivationPrefix` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimDerivationPrefix` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimDerivationSuffix` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimDerivationSuffix` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimSatoshis` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimSatoshis` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`transactionId` <=> NEW.`transactionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,OLD.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`transactionId` <=> NEW.`transactionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 0,NEW.userId,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_0_DELETE AFTER DELETE ON `transactions` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,OLD.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_1_INSERT AFTER INSERT ON `outputs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 1,NEW.userId,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_1_UPDATE AFTER UPDATE ON `outputs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`outputId` AS BINARY) <=> CAST(NEW.`outputId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`basketId` AS BINARY) <=> CAST(NEW.`basketId` AS BINARY)) OR NOT (CAST(OLD.`spendable` AS BINARY) <=> CAST(NEW.`spendable` AS BINARY)) OR NOT (CAST(OLD.`change` AS BINARY) <=> CAST(NEW.`change` AS BINARY)) OR NOT (CAST(OLD.`vout` AS BINARY) <=> CAST(NEW.`vout` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`providedBy` AS BINARY) <=> CAST(NEW.`providedBy` AS BINARY)) OR NOT (CAST(OLD.`purpose` AS BINARY) <=> CAST(NEW.`purpose` AS BINARY)) OR NOT (CAST(OLD.`type` AS BINARY) <=> CAST(NEW.`type` AS BINARY)) OR NOT (CAST(OLD.`outputDescription` AS BINARY) <=> CAST(NEW.`outputDescription` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`senderIdentityKey` AS BINARY) <=> CAST(NEW.`senderIdentityKey` AS BINARY)) OR NOT (CAST(OLD.`derivationPrefix` AS BINARY) <=> CAST(NEW.`derivationPrefix` AS BINARY)) OR NOT (CAST(OLD.`derivationSuffix` AS BINARY) <=> CAST(NEW.`derivationSuffix` AS BINARY)) OR NOT (CAST(OLD.`customInstructions` AS BINARY) <=> CAST(NEW.`customInstructions` AS BINARY)) OR NOT (CAST(OLD.`spentBy` AS BINARY) <=> CAST(NEW.`spentBy` AS BINARY)) OR NOT (CAST(OLD.`sequenceNumber` AS BINARY) <=> CAST(NEW.`sequenceNumber` AS BINARY)) OR NOT (CAST(OLD.`spendingDescription` AS BINARY) <=> CAST(NEW.`spendingDescription` AS BINARY)) OR NOT (CAST(OLD.`scriptLength` AS BINARY) <=> CAST(NEW.`scriptLength` AS BINARY)) OR NOT (CAST(OLD.`scriptOffset` AS BINARY) <=> CAST(NEW.`scriptOffset` AS BINARY)) OR NOT (CAST(OLD.`lockingScript` AS BINARY) <=> CAST(NEW.`lockingScript` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`outputId` <=> NEW.`outputId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,OLD.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`outputId` <=> NEW.`outputId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 1,NEW.userId,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_1_DELETE AFTER DELETE ON `outputs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,OLD.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_2_INSERT AFTER INSERT ON `certificates` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 2,NEW.userId,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_2_UPDATE AFTER UPDATE ON `certificates` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`certificateId` AS BINARY) <=> CAST(NEW.`certificateId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`serialNumber` AS BINARY) <=> CAST(NEW.`serialNumber` AS BINARY)) OR NOT (CAST(OLD.`type` AS BINARY) <=> CAST(NEW.`type` AS BINARY)) OR NOT (CAST(OLD.`certifier` AS BINARY) <=> CAST(NEW.`certifier` AS BINARY)) OR NOT (CAST(OLD.`subject` AS BINARY) <=> CAST(NEW.`subject` AS BINARY)) OR NOT (CAST(OLD.`verifier` AS BINARY) <=> CAST(NEW.`verifier` AS BINARY)) OR NOT (CAST(OLD.`revocationOutpoint` AS BINARY) <=> CAST(NEW.`revocationOutpoint` AS BINARY)) OR NOT (CAST(OLD.`signature` AS BINARY) <=> CAST(NEW.`signature` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`certificateId` <=> NEW.`certificateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,OLD.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`certificateId` <=> NEW.`certificateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 2,NEW.userId,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_2_DELETE AFTER DELETE ON `certificates` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,OLD.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_3_INSERT AFTER INSERT ON `tx_labels` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 3,NEW.userId,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_3_UPDATE AFTER UPDATE ON `tx_labels` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`txLabelId` AS BINARY) <=> CAST(NEW.`txLabelId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`label` AS BINARY) <=> CAST(NEW.`label` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`txLabelId` <=> NEW.`txLabelId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,OLD.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`txLabelId` <=> NEW.`txLabelId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 3,NEW.userId,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_3_DELETE AFTER DELETE ON `tx_labels` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,OLD.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_4_INSERT AFTER INSERT ON `output_baskets` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 4,NEW.userId,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_4_UPDATE AFTER UPDATE ON `output_baskets` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`basketId` AS BINARY) <=> CAST(NEW.`basketId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`name` AS BINARY) <=> CAST(NEW.`name` AS BINARY)) OR NOT (CAST(OLD.`numberOfDesiredUTXOs` AS BINARY) <=> CAST(NEW.`numberOfDesiredUTXOs` AS BINARY)) OR NOT (CAST(OLD.`minimumDesiredUTXOValue` AS BINARY) <=> CAST(NEW.`minimumDesiredUTXOValue` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`basketId` <=> NEW.`basketId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,OLD.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`basketId` <=> NEW.`basketId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 4,NEW.userId,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_4_DELETE AFTER DELETE ON `output_baskets` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,OLD.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_5_INSERT AFTER INSERT ON `output_tags` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 5,NEW.userId,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_5_UPDATE AFTER UPDATE ON `output_tags` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`outputTagId` AS BINARY) <=> CAST(NEW.`outputTagId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`tag` AS BINARY) <=> CAST(NEW.`tag` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`outputTagId` <=> NEW.`outputTagId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,OLD.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`outputTagId` <=> NEW.`outputTagId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 5,NEW.userId,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_5_DELETE AFTER DELETE ON `output_tags` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,OLD.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_6_INSERT AFTER INSERT ON `commissions` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 6,NEW.userId,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_6_UPDATE AFTER UPDATE ON `commissions` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`commissionId` AS BINARY) <=> CAST(NEW.`commissionId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`keyOffset` AS BINARY) <=> CAST(NEW.`keyOffset` AS BINARY)) OR NOT (CAST(OLD.`isRedeemed` AS BINARY) <=> CAST(NEW.`isRedeemed` AS BINARY)) OR NOT (CAST(OLD.`lockingScript` AS BINARY) <=> CAST(NEW.`lockingScript` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`commissionId` <=> NEW.`commissionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,OLD.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`commissionId` <=> NEW.`commissionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 6,NEW.userId,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_6_DELETE AFTER DELETE ON `commissions` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,OLD.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_7_INSERT AFTER INSERT ON `sync_states` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 7,NEW.userId,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_7_UPDATE AFTER UPDATE ON `sync_states` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`syncStateId` AS BINARY) <=> CAST(NEW.`syncStateId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`storageIdentityKey` AS BINARY) <=> CAST(NEW.`storageIdentityKey` AS BINARY)) OR NOT (CAST(OLD.`storageName` AS BINARY) <=> CAST(NEW.`storageName` AS BINARY)) OR NOT (CAST(OLD.`status` AS BINARY) <=> CAST(NEW.`status` AS BINARY)) OR NOT (CAST(OLD.`init` AS BINARY) <=> CAST(NEW.`init` AS BINARY)) OR NOT (CAST(OLD.`refNum` AS BINARY) <=> CAST(NEW.`refNum` AS BINARY)) OR NOT (CAST(OLD.`syncMap` AS BINARY) <=> CAST(NEW.`syncMap` AS BINARY)) OR NOT (CAST(OLD.`when` AS BINARY) <=> CAST(NEW.`when` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`errorLocal` AS BINARY) <=> CAST(NEW.`errorLocal` AS BINARY)) OR NOT (CAST(OLD.`errorOther` AS BINARY) <=> CAST(NEW.`errorOther` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`syncStateId` <=> NEW.`syncStateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,OLD.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`syncStateId` <=> NEW.`syncStateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 7,NEW.userId,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_7_DELETE AFTER DELETE ON `sync_states` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,OLD.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_8_INSERT AFTER INSERT ON `proven_txs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,NEW.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_8_UPDATE AFTER UPDATE ON `proven_txs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`provenTxId` AS BINARY) <=> CAST(NEW.`provenTxId` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`height` AS BINARY) <=> CAST(NEW.`height` AS BINARY)) OR NOT (CAST(OLD.`index` AS BINARY) <=> CAST(NEW.`index` AS BINARY)) OR NOT (CAST(OLD.`merklePath` AS BINARY) <=> CAST(NEW.`merklePath` AS BINARY)) OR NOT (CAST(OLD.`rawTx` AS BINARY) <=> CAST(NEW.`rawTx` AS BINARY)) OR NOT (CAST(OLD.`blockHash` AS BINARY) <=> CAST(NEW.`blockHash` AS BINARY)) OR NOT (CAST(OLD.`merkleRoot` AS BINARY) <=> CAST(NEW.`merkleRoot` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`provenTxId` <=> NEW.`provenTxId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,OLD.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,NEW.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`provenTxId` <=> NEW.`provenTxId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_8_DELETE AFTER DELETE ON `proven_txs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,OLD.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_9_INSERT AFTER INSERT ON `proven_tx_reqs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,NEW.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_9_UPDATE AFTER UPDATE ON `proven_tx_reqs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`provenTxReqId` AS BINARY) <=> CAST(NEW.`provenTxReqId` AS BINARY)) OR NOT (CAST(OLD.`provenTxId` AS BINARY) <=> CAST(NEW.`provenTxId` AS BINARY)) OR NOT (CAST(OLD.`status` AS BINARY) <=> CAST(NEW.`status` AS BINARY)) OR NOT (CAST(OLD.`attempts` AS BINARY) <=> CAST(NEW.`attempts` AS BINARY)) OR NOT (CAST(OLD.`notified` AS BINARY) <=> CAST(NEW.`notified` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`batch` AS BINARY) <=> CAST(NEW.`batch` AS BINARY)) OR NOT (CAST(OLD.`history` AS BINARY) <=> CAST(NEW.`history` AS BINARY)) OR NOT (CAST(OLD.`notify` AS BINARY) <=> CAST(NEW.`notify` AS BINARY)) OR NOT (CAST(OLD.`rawTx` AS BINARY) <=> CAST(NEW.`rawTx` AS BINARY)) OR NOT (CAST(OLD.`inputBEEF` AS BINARY) <=> CAST(NEW.`inputBEEF` AS BINARY)) OR NOT (CAST(OLD.`wasBroadcast` AS BINARY) <=> CAST(NEW.`wasBroadcast` AS BINARY)) OR NOT (CAST(OLD.`rebroadcastAttempts` AS BINARY) <=> CAST(NEW.`rebroadcastAttempts` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`provenTxReqId` <=> NEW.`provenTxReqId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,OLD.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,NEW.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`provenTxReqId` <=> NEW.`provenTxReqId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_9_DELETE AFTER DELETE ON `proven_tx_reqs` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,OLD.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_10_INSERT AFTER INSERT ON `tx_labels_map` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.txLabelId BETWEEN 1 AND 9007199254740991) AND (NEW.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,NEW.txLabelId,NEW.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 10,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=0 AND snapshotLeftId=NEW.txLabelId AND snapshotRightId=NEW.transactionId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_10_UPDATE AFTER UPDATE ON `tx_labels_map` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`txLabelId` AS BINARY) <=> CAST(NEW.`txLabelId` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.txLabelId BETWEEN 1 AND 9007199254740991) AND (NEW.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.txLabelId BETWEEN 1 AND 9007199254740991) AND (OLD.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.txLabelId <=> NEW.txLabelId) AND (OLD.transactionId <=> NEW.transactionId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,OLD.txLabelId,OLD.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,NEW.txLabelId,NEW.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.txLabelId <=> NEW.txLabelId) AND (OLD.transactionId <=> NEW.transactionId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 10,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=0 AND snapshotLeftId=NEW.txLabelId AND snapshotRightId=NEW.transactionId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_10_DELETE AFTER DELETE ON `tx_labels_map` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.txLabelId BETWEEN 1 AND 9007199254740991) AND (OLD.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,OLD.txLabelId,OLD.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_11_INSERT AFTER INSERT ON `output_tags_map` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.outputTagId BETWEEN 1 AND 9007199254740991) AND (NEW.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,NEW.outputTagId,NEW.outputId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 11,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=1 AND snapshotLeftId=NEW.outputTagId AND snapshotRightId=NEW.outputId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_11_UPDATE AFTER UPDATE ON `output_tags_map` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`outputTagId` AS BINARY) <=> CAST(NEW.`outputTagId` AS BINARY)) OR NOT (CAST(OLD.`outputId` AS BINARY) <=> CAST(NEW.`outputId` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.outputTagId BETWEEN 1 AND 9007199254740991) AND (NEW.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.outputTagId BETWEEN 1 AND 9007199254740991) AND (OLD.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.outputTagId <=> NEW.outputTagId) AND (OLD.outputId <=> NEW.outputId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,OLD.outputTagId,OLD.outputId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,NEW.outputTagId,NEW.outputId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.outputTagId <=> NEW.outputTagId) AND (OLD.outputId <=> NEW.outputId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 11,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=1 AND snapshotLeftId=NEW.outputTagId AND snapshotRightId=NEW.outputId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_11_DELETE AFTER DELETE ON `output_tags_map` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.outputTagId BETWEEN 1 AND 9007199254740991) AND (OLD.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,OLD.outputTagId,OLD.outputId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_12_INSERT AFTER INSERT ON `certificate_fields` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,NEW.certificateId,0,CAST(NEW.fieldName AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,snapshotUserId,snapshotCertificateId,0,CAST(snapshotFieldName AS BINARY),journalRevision,1 FROM snapshot_certificate_field_keys WHERE snapshotCertificateId=NEW.certificateId AND snapshotFieldName=NEW.fieldName AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_12_UPDATE AFTER UPDATE ON `certificate_fields` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`certificateId` AS BINARY) <=> CAST(NEW.`certificateId` AS BINARY)) OR NOT (CAST(OLD.`fieldName` AS BINARY) <=> CAST(NEW.`fieldName` AS BINARY)) OR NOT (CAST(OLD.`fieldValue` AS BINARY) <=> CAST(NEW.`fieldValue` AS BINARY)) OR NOT (CAST(OLD.`masterKey` AS BINARY) <=> CAST(NEW.`masterKey` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(OLD.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.certificateId <=> NEW.certificateId) AND (0 <=> 0) AND (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,OLD.certificateId,0,CAST(OLD.fieldName AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,NEW.certificateId,0,CAST(NEW.fieldName AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.certificateId <=> NEW.certificateId) AND (0 <=> 0) AND (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,snapshotUserId,snapshotCertificateId,0,CAST(snapshotFieldName AS BINARY),journalRevision,1 FROM snapshot_certificate_field_keys WHERE snapshotCertificateId=NEW.certificateId AND snapshotFieldName=NEW.fieldName AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "bindings": [] + }, + { + "sql": "CREATE TRIGGER snapshot_journal_physical_12_DELETE AFTER DELETE ON `certificate_fields` FOR EACH ROW BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(OLD.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,OLD.certificateId,0,CAST(OLD.fieldName AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "bindings": [] + } + ] +} diff --git a/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-metadata-fixture.json b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-metadata-fixture.json new file mode 100644 index 000000000..01b946a94 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-metadata-fixture.json @@ -0,0 +1,13288 @@ +[ + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["knex_migrations"] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_profile_keys"] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_profile_index_progress"] + }, + { + "sql": "SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_profile_keys"], + "rows": [ + { + "name": "snapshotTableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "snapshotUserId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "snapshotRowId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX", + "bindings": ["snapshot_profile_keys"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "snapshotRowId", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_profile_index_progress"], + "rows": [ + { + "name": "snapshotTableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "afterRowId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "complete", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX", + "bindings": ["snapshot_profile_index_progress"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_relation_keys"] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_relation_index_progress"] + }, + { + "sql": "SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_relation_keys"], + "rows": [ + { + "name": "snapshotTableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "snapshotUserId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "snapshotLeftId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "snapshotRightId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "snapshotMembership", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX", + "bindings": ["snapshot_relation_keys"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "snapshotLeftId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "snapshotRightId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_map", + "columnName": "snapshotTableId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_map", + "columnName": "snapshotLeftId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_map", + "columnName": "snapshotRightId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_map", + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_right", + "columnName": "snapshotTableId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_right", + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_right", + "columnName": "snapshotRightId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_relation_right", + "columnName": "snapshotLeftId", + "nonUnique": 1, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_relation_index_progress"], + "rows": [ + { + "name": "snapshotTableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "afterLeftId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "afterRightId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "complete", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX", + "bindings": ["snapshot_relation_index_progress"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_certificate_field_keys"] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_certificate_index_progress"] + }, + { + "sql": "SELECT TABLE_NAME AS name, ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME IN (?, ?) ORDER BY TABLE_NAME", + "bindings": ["certificate_fields", "certificates"], + "rows": [ + { + "name": "certificate_fields", + "engine": "InnoDB" + }, + { + "name": "certificates", + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT COLUMN_TYPE AS type, IS_NULLABLE AS nullable, CHARACTER_SET_NAME AS charset, COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? AND COLUMN_NAME = ?", + "bindings": ["certificate_fields", "fieldName"], + "rows": [ + { + "type": "varchar(100)", + "nullable": "NO", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + } + ] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?", + "bindings": ["snapshot_certificate_field_keys"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra, CHARACTER_SET_NAME AS charset, COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_certificate_field_keys"], + "rows": [ + { + "name": "snapshotUserId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "snapshotFieldName", + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "snapshotCertificateId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "snapshotMembership", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX", + "bindings": ["snapshot_certificate_field_keys"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "snapshotFieldName", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "snapshotCertificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_certificate_lookup", + "columnName": "snapshotFieldName", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_certificate_lookup", + "columnName": "snapshotCertificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_certificate_lookup", + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_certificate_parent", + "columnName": "snapshotCertificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_certificate_parent", + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_certificate_parent", + "columnName": "snapshotFieldName", + "nonUnique": 1, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ?", + "bindings": ["snapshot_certificate_index_progress"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name, COLUMN_TYPE AS type, IS_NULLABLE AS nullable, COLUMN_DEFAULT AS defaultValue, EXTRA AS extra, CHARACTER_SET_NAME AS charset, COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_certificate_index_progress"], + "rows": [ + { + "name": "snapshotTableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "started", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "afterFieldName", + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "afterCertificateId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "complete", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA = DATABASE() AND TABLE_NAME = ? ORDER BY INDEX_NAME, SEQ_IN_INDEX", + "bindings": ["snapshot_certificate_index_progress"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["proven_txs"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT UPDATE_RULE AS updateRule,DELETE_RULE AS deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["proven_txs"], + "rows": [] + }, + { + "sql": "SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,EXTRA AS extra,CHARACTER_SET_NAME AS charset,COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["proven_txs"], + "rows": [ + { + "name": "blockHash", + "type": "varchar(64)", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null + }, + { + "name": "height", + "type": "int unsigned", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "index", + "type": "int unsigned", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "merklePath", + "type": "blob", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "merkleRoot", + "type": "varchar(64)", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "provenTxId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment", + "charset": null, + "collation": null + }, + { + "name": "rawTx", + "type": "longblob", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "txid", + "type": "varchar(64)", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["proven_txs"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "provenTxId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "proven_txs_blockhash_index", + "columnName": "blockHash", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "proven_txs_txid_unique", + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["proven_tx_reqs"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT UPDATE_RULE AS updateRule,DELETE_RULE AS deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["proven_tx_reqs"], + "rows": [ + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,EXTRA AS extra,CHARACTER_SET_NAME AS charset,COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["proven_tx_reqs"], + "rows": [ + { + "name": "attempts", + "type": "int unsigned", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "batch", + "type": "varchar(64)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null + }, + { + "name": "history", + "type": "longtext", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "inputBEEF", + "type": "longblob", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "notified", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "notify", + "type": "longtext", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "provenTxId", + "type": "int unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "provenTxReqId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment", + "charset": null, + "collation": null + }, + { + "name": "rawTx", + "type": "longblob", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "rebroadcastAttempts", + "type": "int unsigned", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "status", + "type": "varchar(16)", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "txid", + "type": "varchar(64)", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null + }, + { + "name": "wasBroadcast", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["proven_tx_reqs"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "provenTxReqId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "proven_tx_reqs_batch_index", + "columnName": "batch", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "proven_tx_reqs_proventxid_foreign", + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "proven_tx_reqs_status_index", + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "proven_tx_reqs_txid_index", + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "proven_tx_reqs_txid_unique", + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["transactions"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT UPDATE_RULE AS updateRule,DELETE_RULE AS deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["transactions"], + "rows": [ + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,EXTRA AS extra,CHARACTER_SET_NAME AS charset,COLLATION_NAME AS collation FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["transactions"], + "rows": [ + { + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null + }, + { + "name": "description", + "type": "varchar(2048)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "inputBEEF", + "type": "longblob", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "isOutgoing", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "lockTime", + "type": "int unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "noSendExpiryAnchorTxid", + "type": "varchar(64)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "noSendExpiryAnchorVout", + "type": "int unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "noSendExpiryDeadline", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "noSendExpiryMode", + "type": "varchar(16)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "noSendExpiryObservedAt", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "noSendExpiryReclaimDerivationPrefix", + "type": "varchar(32)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "noSendExpiryReclaimDerivationSuffix", + "type": "varchar(32)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "noSendExpiryReclaimRawTx", + "type": "longblob", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "noSendExpiryReclaimSatoshis", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "noSendExpiryReclaimTxid", + "type": "varchar(64)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "noSendExpiryReleasedAt", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "noSendExpiryState", + "type": "varchar(24)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "noSendExpiryValue", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "provenTxId", + "type": "int unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "rawTx", + "type": "longblob", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "reference", + "type": "varchar(64)", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "satoshis", + "type": "bigint", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "status", + "type": "varchar(64)", + "nullable": "NO", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "transactionId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment", + "charset": null, + "collation": null + }, + { + "name": "txid", + "type": "varchar(64)", + "nullable": "YES", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci" + }, + { + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null + }, + { + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "", + "charset": null, + "collation": null + }, + { + "name": "version", + "type": "int unsigned", + "nullable": "YES", + "extra": "", + "charset": null, + "collation": null + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["transactions"], + "rows": [ + { + "name": "idx_transactions_nosend_expiry", + "columnName": "noSendExpiryState", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "idx_transactions_nosend_expiry", + "columnName": "noSendExpiryDeadline", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "idx_transactions_nosend_reclaim", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "idx_transactions_nosend_reclaim", + "columnName": "noSendExpiryReclaimTxid", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "idx_transactions_user_proven_tx", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "idx_transactions_user_proven_tx", + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "idx_transactions_user_txid", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "idx_transactions_user_txid", + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "transactions_proventxid_foreign", + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "transactions_reference_unique", + "columnName": "reference", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "transactions_status_index", + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "transactions_txid_index", + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_global_guards"] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_global_guards"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,COLUMN_DEFAULT AS defaultValue,EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_global_guards"], + "rows": [ + { + "name": "proofId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "present", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_global_guards"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "proofId", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_global_keys"] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_global_keys"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,COLUMN_DEFAULT AS defaultValue,EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_global_keys"], + "rows": [ + { + "name": "tableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "rowId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "refs", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "present", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_global_keys"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "tableId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "rowId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_page", + "columnName": "tableId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_page", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_page", + "columnName": "present", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_page", + "columnName": "rowId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_target", + "columnName": "tableId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_target", + "columnName": "rowId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_target", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_global_edges"] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_global_edges"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,COLUMN_DEFAULT AS defaultValue,EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_global_edges"], + "rows": [ + { + "name": "transactionId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "requestId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "tableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "rowId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_global_edges"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "requestId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "tableId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "PRIMARY", + "columnName": "rowId", + "nonUnique": 0, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_request", + "columnName": "requestId", + "nonUnique": 1, + "direction": "A", + "prefix": null + }, + { + "name": "snapshot_global_request", + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "select * from information_schema.tables where table_name = ? and table_schema = database()", + "bindings": ["snapshot_global_index_progress"] + }, + { + "sql": "SELECT ENGINE AS engine FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_global_index_progress"], + "rows": [ + { + "engine": "InnoDB" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME AS name,COLUMN_TYPE AS type,IS_NULLABLE AS nullable,COLUMN_DEFAULT AS defaultValue,EXTRA AS extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["snapshot_global_index_progress"], + "rows": [ + { + "name": "id", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "afterRowId", + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "" + }, + { + "name": "complete", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME AS name, COLUMN_NAME AS columnName, NON_UNIQUE AS nonUnique, COLLATION AS direction, SUB_PART AS prefix FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_global_index_progress"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "id", + "nonUnique": 0, + "direction": "A", + "prefix": null + } + ] + }, + { + "sql": "SELECT TABLE_NAME name,ENGINE engine,TABLE_TYPE type FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields" + ], + "rows": [ + { + "name": "certificate_fields", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "certificates", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "commissions", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "output_baskets", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "output_tags", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "output_tags_map", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "outputs", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "proven_tx_reqs", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "proven_txs", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "sync_states", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "transactions", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "tx_labels", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "tx_labels_map", + "engine": "InnoDB", + "type": "BASE TABLE" + } + ] + }, + { + "sql": "SELECT TABLE_NAME tableName,COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,EXTRA extra FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?) ORDER BY TABLE_NAME,ORDINAL_POSITION LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields" + ], + "rows": [ + { + "tableName": "certificate_fields", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "certificate_fields", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "certificate_fields", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificate_fields", + "name": "certificateId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificate_fields", + "name": "fieldName", + "type": "varchar(100)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificate_fields", + "name": "fieldValue", + "type": "varchar(255)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificate_fields", + "name": "masterKey", + "type": "varchar(255)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "certificates", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "certificates", + "name": "certificateId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "certificates", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "serialNumber", + "type": "varchar(100)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "type", + "type": "varchar(100)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "certifier", + "type": "varchar(100)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "subject", + "type": "varchar(100)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "verifier", + "type": "varchar(100)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "certificates", + "name": "revocationOutpoint", + "type": "varchar(100)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "signature", + "type": "varchar(255)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "certificates", + "name": "isDeleted", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "commissions", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "commissions", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "commissions", + "name": "commissionId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "commissions", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "commissions", + "name": "transactionId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "commissions", + "name": "satoshis", + "type": "int", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "commissions", + "name": "keyOffset", + "type": "varchar(130)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "commissions", + "name": "isRedeemed", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "commissions", + "name": "lockingScript", + "type": "blob", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_baskets", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "output_baskets", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "output_baskets", + "name": "basketId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "output_baskets", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_baskets", + "name": "name", + "type": "varchar(300)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_baskets", + "name": "numberOfDesiredUTXOs", + "type": "int", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_baskets", + "name": "minimumDesiredUTXOValue", + "type": "int", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_baskets", + "name": "isDeleted", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_tags", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "output_tags", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "output_tags", + "name": "outputTagId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "output_tags", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_tags", + "name": "tag", + "type": "varchar(150)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_tags", + "name": "isDeleted", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_tags_map", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "output_tags_map", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "output_tags_map", + "name": "outputTagId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_tags_map", + "name": "outputId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "output_tags_map", + "name": "isDeleted", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "outputs", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "outputs", + "name": "outputId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "outputs", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "transactionId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "basketId", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "spendable", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "change", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "vout", + "type": "int", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "satoshis", + "type": "bigint", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "providedBy", + "type": "varchar(130)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "purpose", + "type": "varchar(20)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "type", + "type": "varchar(50)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "outputs", + "name": "outputDescription", + "type": "varchar(2048)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "txid", + "type": "varchar(64)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "senderIdentityKey", + "type": "varchar(130)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "derivationPrefix", + "type": "varchar(200)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "derivationSuffix", + "type": "varchar(200)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "customInstructions", + "type": "varchar(2500)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "spentBy", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "sequenceNumber", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "spendingDescription", + "type": "varchar(2048)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "scriptLength", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "scriptOffset", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "outputs", + "name": "lockingScript", + "type": "longblob", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "proven_tx_reqs", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "proven_tx_reqs", + "name": "provenTxReqId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "proven_tx_reqs", + "name": "provenTxId", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "status", + "type": "varchar(16)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "attempts", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "notified", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "txid", + "type": "varchar(64)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "batch", + "type": "varchar(64)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "history", + "type": "longtext", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "notify", + "type": "longtext", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "rawTx", + "type": "longblob", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "inputBEEF", + "type": "longblob", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "wasBroadcast", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "rebroadcastAttempts", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_txs", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "proven_txs", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "proven_txs", + "name": "provenTxId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "proven_txs", + "name": "txid", + "type": "varchar(64)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_txs", + "name": "height", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_txs", + "name": "index", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_txs", + "name": "merklePath", + "type": "blob", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_txs", + "name": "rawTx", + "type": "longblob", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "proven_txs", + "name": "blockHash", + "type": "varchar(64)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "proven_txs", + "name": "merkleRoot", + "type": "varchar(64)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "sync_states", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "sync_states", + "name": "syncStateId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "sync_states", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "storageIdentityKey", + "type": "varchar(130)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "storageName", + "type": "varchar(255)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "status", + "type": "varchar(255)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "init", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "refNum", + "type": "varchar(100)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "syncMap", + "type": "longtext", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "when", + "type": "datetime", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "satoshis", + "type": "bigint", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "errorLocal", + "type": "longtext", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "sync_states", + "name": "errorOther", + "type": "longtext", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "transactions", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "transactions", + "name": "transactionId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "transactions", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "transactions", + "name": "provenTxId", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "status", + "type": "varchar(64)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "transactions", + "name": "reference", + "type": "varchar(64)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "transactions", + "name": "isOutgoing", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "transactions", + "name": "satoshis", + "type": "bigint", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "transactions", + "name": "version", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "lockTime", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "description", + "type": "varchar(2048)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "txid", + "type": "varchar(64)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "inputBEEF", + "type": "longblob", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "rawTx", + "type": "longblob", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryMode", + "type": "varchar(16)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryValue", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryDeadline", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryState", + "type": "varchar(24)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryAnchorTxid", + "type": "varchar(64)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryAnchorVout", + "type": "int unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReleasedAt", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryObservedAt", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimTxid", + "type": "varchar(64)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimRawTx", + "type": "longblob", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimDerivationPrefix", + "type": "varchar(32)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimDerivationSuffix", + "type": "varchar(32)", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimSatoshis", + "type": "bigint unsigned", + "nullable": "YES", + "extra": "" + }, + { + "tableName": "tx_labels", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "tx_labels", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "tx_labels", + "name": "txLabelId", + "type": "int unsigned", + "nullable": "NO", + "extra": "auto_increment" + }, + { + "tableName": "tx_labels", + "name": "userId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "tx_labels", + "name": "label", + "type": "varchar(300)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "tx_labels", + "name": "isDeleted", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "tx_labels_map", + "name": "created_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "tx_labels_map", + "name": "updated_at", + "type": "timestamp(3)", + "nullable": "NO", + "extra": "DEFAULT_GENERATED" + }, + { + "tableName": "tx_labels_map", + "name": "txLabelId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "tx_labels_map", + "name": "transactionId", + "type": "int unsigned", + "nullable": "NO", + "extra": "" + }, + { + "tableName": "tx_labels_map", + "name": "isDeleted", + "type": "tinyint(1)", + "nullable": "NO", + "extra": "" + } + ] + }, + { + "sql": "SELECT TABLE_NAME tableName,INDEX_NAME name,COLUMN_NAME columnName,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,IS_VISIBLE visible FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?) ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields" + ], + "rows": [ + { + "tableName": "certificate_fields", + "name": "certificate_fields_certificateid_foreign", + "columnName": "certificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_fieldname_certificateid_unique", + "columnName": "fieldName", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_fieldname_certificateid_unique", + "columnName": "certificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_userid_foreign", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "columnName": "type", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "columnName": "certifier", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "columnName": "serialNumber", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "certificates", + "name": "PRIMARY", + "columnName": "certificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_index", + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_unique", + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "commissions", + "name": "commissions_userid_foreign", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "commissions", + "name": "PRIMARY", + "columnName": "commissionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_baskets", + "name": "output_baskets_name_userid_unique", + "columnName": "name", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_baskets", + "name": "output_baskets_name_userid_unique", + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_baskets", + "name": "output_baskets_userid_foreign", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_baskets", + "name": "PRIMARY", + "columnName": "basketId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags", + "name": "output_tags_tag_userid_unique", + "columnName": "tag", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags", + "name": "output_tags_tag_userid_unique", + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags", + "name": "output_tags_userid_foreign", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags", + "name": "PRIMARY", + "columnName": "outputTagId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "columnName": "isDeleted", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "columnName": "outputTagId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputid_index", + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_outputid_unique", + "columnName": "outputTagId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_outputid_unique", + "columnName": "outputId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "columnName": "spentBy", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "columnName": "satoshis", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_spentby", + "columnName": "spentBy", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "columnName": "satoshis", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "outputs_basketid_foreign", + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "outputs_spendable_index", + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "columnName": "vout", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "outputs", + "name": "PRIMARY", + "columnName": "outputId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_tx_reqs", + "name": "PRIMARY", + "columnName": "provenTxReqId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_batch_index", + "columnName": "batch", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_proventxid_foreign", + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_status_index", + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_txid_index", + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_txid_unique", + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_txs", + "name": "PRIMARY", + "columnName": "provenTxId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_txs", + "name": "proven_txs_blockhash_index", + "columnName": "blockHash", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "proven_txs", + "name": "proven_txs_txid_unique", + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "sync_states", + "name": "PRIMARY", + "columnName": "syncStateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "sync_states", + "name": "sync_states_refnum_index", + "columnName": "refNum", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "sync_states", + "name": "sync_states_refnum_unique", + "columnName": "refNum", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "sync_states", + "name": "sync_states_status_index", + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "sync_states", + "name": "sync_states_userid_foreign", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_expiry", + "columnName": "noSendExpiryState", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_expiry", + "columnName": "noSendExpiryDeadline", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_reclaim", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_reclaim", + "columnName": "noSendExpiryReclaimTxid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_proven_tx", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_proven_tx", + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_txid", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_txid", + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "PRIMARY", + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "transactions_proventxid_foreign", + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "transactions_reference_unique", + "columnName": "reference", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "transactions_status_index", + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "transactions", + "name": "transactions_txid_index", + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels", + "name": "PRIMARY", + "columnName": "txLabelId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels", + "name": "tx_labels_label_userid_unique", + "columnName": "label", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels", + "name": "tx_labels_label_userid_unique", + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels", + "name": "tx_labels_userid_foreign", + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels_map", + "name": "idx_tx_labels_map_tx_deleted", + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels_map", + "name": "idx_tx_labels_map_tx_deleted", + "columnName": "isDeleted", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_transactionid_index", + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_transactionid_unique", + "columnName": "txLabelId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_transactionid_unique", + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "visible": "YES" + } + ] + }, + { + "sql": "SELECT UPDATE_RULE updateRule,DELETE_RULE deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?)", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields" + ], + "rows": [ + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)", + "bindings": [ + 1873, + "snapshot_profile_0_insert", + "snapshot_profile_0_update", + "snapshot_profile_0_delete", + "snapshot_profile_1_insert", + "snapshot_profile_1_update", + "snapshot_profile_1_delete", + "snapshot_profile_2_insert", + "snapshot_profile_2_update", + "snapshot_profile_2_delete", + "snapshot_profile_3_insert", + "snapshot_profile_3_update", + "snapshot_profile_3_delete", + "snapshot_profile_4_insert", + "snapshot_profile_4_update", + "snapshot_profile_4_delete", + "snapshot_profile_5_insert", + "snapshot_profile_5_update", + "snapshot_profile_5_delete", + "snapshot_profile_6_insert", + "snapshot_profile_6_update", + "snapshot_profile_6_delete", + "snapshot_profile_7_insert", + "snapshot_profile_7_update", + "snapshot_profile_7_delete", + "snapshot_relation_0_map_delete", + "snapshot_relation_0_map_before_update", + "snapshot_relation_0_left_delete", + "snapshot_relation_0_left_before_update", + "snapshot_relation_0_right_delete", + "snapshot_relation_0_right_before_update", + "snapshot_relation_1_map_delete", + "snapshot_relation_1_map_before_update", + "snapshot_relation_1_left_delete", + "snapshot_relation_1_left_before_update", + "snapshot_relation_1_right_delete", + "snapshot_relation_1_right_before_update", + "snapshot_relation_0_map_insert", + "snapshot_relation_0_map_after_update", + "snapshot_relation_0_left_insert", + "snapshot_relation_0_left_after_update", + "snapshot_relation_0_right_insert", + "snapshot_relation_0_right_after_update", + "snapshot_relation_1_map_insert", + "snapshot_relation_1_map_after_update", + "snapshot_relation_1_left_insert", + "snapshot_relation_1_left_after_update", + "snapshot_relation_1_right_insert", + "snapshot_relation_1_right_after_update", + "snapshot_certificate_field_delete", + "snapshot_certificate_field_before_update", + "snapshot_certificate_parent_delete", + "snapshot_certificate_parent_before_update", + "snapshot_certificate_field_insert", + "snapshot_certificate_field_after_update", + "snapshot_certificate_parent_insert", + "snapshot_certificate_parent_after_update", + "snapshot_global_edge_delete", + "snapshot_global_edge_insert", + "snapshot_global_tx_delete", + "snapshot_global_tx_before_update", + "snapshot_global_req_delete", + "snapshot_global_req_before_update", + "snapshot_global_proof_delete", + "snapshot_global_proof_before_update", + "snapshot_global_proof_insert", + "snapshot_global_proof_after_update", + "snapshot_global_tx_insert", + "snapshot_global_tx_after_update", + "snapshot_global_req_insert", + "snapshot_global_req_after_update" + ], + "rows": [ + { + "name": "snapshot_certificate_field_insert", + "table": "certificate_fields", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END" + }, + { + "name": "snapshot_certificate_field_before_update", + "table": "certificate_fields", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END IF; END" + }, + { + "name": "snapshot_certificate_field_after_update", + "table": "certificate_fields", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_certificate_field_delete", + "table": "certificate_fields", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END" + }, + { + "name": "snapshot_profile_2_insert", + "table": "certificates", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END" + }, + { + "name": "snapshot_certificate_parent_insert", + "table": "certificates", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END" + }, + { + "name": "snapshot_certificate_parent_before_update", + "table": "certificates", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_profile_2_update", + "table": "certificates", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END IF; END" + }, + { + "name": "snapshot_certificate_parent_after_update", + "table": "certificates", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_profile_2_delete", + "table": "certificates", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; END" + }, + { + "name": "snapshot_certificate_parent_delete", + "table": "certificates", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_profile_6_insert", + "table": "commissions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END" + }, + { + "name": "snapshot_profile_6_update", + "table": "commissions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.commissionId <=> NEW.commissionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END IF; END" + }, + { + "name": "snapshot_profile_6_delete", + "table": "commissions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; END" + }, + { + "name": "snapshot_profile_4_insert", + "table": "output_baskets", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END" + }, + { + "name": "snapshot_profile_4_update", + "table": "output_baskets", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.basketId <=> NEW.basketId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END IF; END" + }, + { + "name": "snapshot_profile_4_delete", + "table": "output_baskets", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; END" + }, + { + "name": "snapshot_profile_5_insert", + "table": "output_tags", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END" + }, + { + "name": "snapshot_relation_1_left_insert", + "table": "output_tags", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END" + }, + { + "name": "snapshot_relation_1_left_before_update", + "table": "output_tags", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_profile_5_update", + "table": "output_tags", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputTagId <=> NEW.outputTagId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END IF; END" + }, + { + "name": "snapshot_relation_1_left_after_update", + "table": "output_tags", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END" + }, + { + "name": "snapshot_profile_5_delete", + "table": "output_tags", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; END" + }, + { + "name": "snapshot_relation_1_left_delete", + "table": "output_tags", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_relation_1_map_insert", + "table": "output_tags_map", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_relation_1_map_before_update", + "table": "output_tags_map", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END IF; END" + }, + { + "name": "snapshot_relation_1_map_after_update", + "table": "output_tags_map", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_relation_1_map_delete", + "table": "output_tags_map", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END" + }, + { + "name": "snapshot_profile_1_insert", + "table": "outputs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END" + }, + { + "name": "snapshot_relation_1_right_insert", + "table": "outputs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_relation_1_right_before_update", + "table": "outputs", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_profile_1_update", + "table": "outputs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END IF; END" + }, + { + "name": "snapshot_relation_1_right_after_update", + "table": "outputs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_profile_1_delete", + "table": "outputs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; END" + }, + { + "name": "snapshot_relation_1_right_delete", + "table": "outputs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_global_req_insert", + "table": "proven_tx_reqs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END" + }, + { + "name": "snapshot_global_req_before_update", + "table": "proven_tx_reqs", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END IF; END" + }, + { + "name": "snapshot_global_req_after_update", + "table": "proven_tx_reqs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END" + }, + { + "name": "snapshot_global_req_delete", + "table": "proven_tx_reqs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END" + }, + { + "name": "snapshot_global_proof_insert", + "table": "proven_txs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END" + }, + { + "name": "snapshot_global_proof_before_update", + "table": "proven_txs", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END IF; END" + }, + { + "name": "snapshot_global_proof_after_update", + "table": "proven_txs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END" + }, + { + "name": "snapshot_global_proof_delete", + "table": "proven_txs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END" + }, + { + "name": "snapshot_global_edge_insert", + "table": "snapshot_global_edges", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_global_keys (tableId,userId,rowId,refs,present) VALUES (NEW.tableId,NEW.userId,NEW.rowId,1,CASE WHEN NEW.tableId=0 THEN 1 ELSE (SELECT present FROM snapshot_global_guards WHERE proofId=NEW.rowId FOR SHARE) END) ON DUPLICATE KEY UPDATE refs=snapshot_global_keys.refs+1; END" + }, + { + "name": "snapshot_global_edge_delete", + "table": "snapshot_global_edges", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_global_keys SET refs=refs-1 WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId; DELETE FROM snapshot_global_keys WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId AND refs=0; DELETE FROM snapshot_global_guards WHERE proofId=OLD.rowId AND OLD.tableId=1 AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.rowId); END" + }, + { + "name": "snapshot_profile_7_insert", + "table": "sync_states", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END" + }, + { + "name": "snapshot_profile_7_update", + "table": "sync_states", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.syncStateId <=> NEW.syncStateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END IF; END" + }, + { + "name": "snapshot_profile_7_delete", + "table": "sync_states", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; END" + }, + { + "name": "snapshot_profile_0_insert", + "table": "transactions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END" + }, + { + "name": "snapshot_relation_0_right_insert", + "table": "transactions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_global_tx_insert", + "table": "transactions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END" + }, + { + "name": "snapshot_relation_0_right_before_update", + "table": "transactions", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_global_tx_before_update", + "table": "transactions", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END IF; END" + }, + { + "name": "snapshot_profile_0_update", + "table": "transactions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END IF; END" + }, + { + "name": "snapshot_relation_0_right_after_update", + "table": "transactions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_global_tx_after_update", + "table": "transactions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END IF; END" + }, + { + "name": "snapshot_profile_0_delete", + "table": "transactions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; END" + }, + { + "name": "snapshot_relation_0_right_delete", + "table": "transactions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_global_tx_delete", + "table": "transactions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END" + }, + { + "name": "snapshot_profile_3_insert", + "table": "tx_labels", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END" + }, + { + "name": "snapshot_relation_0_left_insert", + "table": "tx_labels", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END" + }, + { + "name": "snapshot_relation_0_left_before_update", + "table": "tx_labels", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_profile_3_update", + "table": "tx_labels", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txLabelId <=> NEW.txLabelId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END IF; END" + }, + { + "name": "snapshot_relation_0_left_after_update", + "table": "tx_labels", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END" + }, + { + "name": "snapshot_profile_3_delete", + "table": "tx_labels", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; END" + }, + { + "name": "snapshot_relation_0_left_delete", + "table": "tx_labels", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_relation_0_map_insert", + "table": "tx_labels_map", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_relation_0_map_before_update", + "table": "tx_labels_map", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END IF; END" + }, + { + "name": "snapshot_relation_0_map_after_update", + "table": "tx_labels_map", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_relation_0_map_delete", + "table": "tx_labels_map", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END" + } + ] + }, + { + "sql": "SELECT TABLE_NAME name,ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,SUBSTRING(CREATE_OPTIONS,1,16385) options FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ORDER BY TABLE_NAME LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields", + "users", + "settings", + "snapshot_profile_keys", + "snapshot_relation_keys", + "snapshot_certificate_field_keys", + "snapshot_global_edges", + "snapshot_global_keys", + "snapshot_global_guards" + ], + "rows": [ + { + "name": "certificate_fields", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "certificates", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "commissions", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "output_baskets", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "output_tags", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "output_tags_map", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "outputs", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "proven_tx_reqs", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "proven_txs", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "settings", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_certificate_field_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_global_edges", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_global_guards", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_global_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_profile_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_relation_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "sync_states", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "transactions", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "tx_labels", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "tx_labels_map", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "users", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + } + ] + }, + { + "sql": "SELECT TABLE_NAME tableName,COLUMN_NAME name,ORDINAL_POSITION position,COLUMN_TYPE type,IS_NULLABLE nullable,SUBSTRING(COLUMN_DEFAULT,1,16385) defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,SUBSTRING(GENERATION_EXPRESSION,1,16385) expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ORDER BY TABLE_NAME,ORDINAL_POSITION LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields", + "users", + "settings", + "snapshot_profile_keys", + "snapshot_relation_keys", + "snapshot_certificate_field_keys", + "snapshot_global_edges", + "snapshot_global_keys", + "snapshot_global_guards" + ], + "rows": [ + { + "tableName": "certificate_fields", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "userId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "certificateId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "fieldName", + "position": 5, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "fieldValue", + "position": 6, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "masterKey", + "position": 7, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": "", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "certificateId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "serialNumber", + "position": 5, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "type", + "position": 6, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "certifier", + "position": 7, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "subject", + "position": 8, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "verifier", + "position": 9, + "type": "varchar(100)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "revocationOutpoint", + "position": 10, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "signature", + "position": 11, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "isDeleted", + "position": 12, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "commissionId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "transactionId", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "satoshis", + "position": 6, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "keyOffset", + "position": 7, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "commissions", + "name": "isRedeemed", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "lockingScript", + "position": 9, + "type": "blob", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "basketId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "name", + "position": 5, + "type": "varchar(300)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "numberOfDesiredUTXOs", + "position": 6, + "type": "int", + "nullable": "NO", + "defaultValue": "144", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "minimumDesiredUTXOValue", + "position": 7, + "type": "int", + "nullable": "NO", + "defaultValue": "5000", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "isDeleted", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "outputTagId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "tag", + "position": 5, + "type": "varchar(150)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "output_tags", + "name": "isDeleted", + "position": 6, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "outputTagId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "outputId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "isDeleted", + "position": 5, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "outputId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "transactionId", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "basketId", + "position": 6, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "spendable", + "position": 7, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "change", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "vout", + "position": 9, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "satoshis", + "position": 10, + "type": "bigint", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "providedBy", + "position": 11, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "purpose", + "position": 12, + "type": "varchar(20)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "type", + "position": 13, + "type": "varchar(50)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "outputDescription", + "position": 14, + "type": "varchar(2048)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "txid", + "position": 15, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "senderIdentityKey", + "position": 16, + "type": "varchar(130)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "derivationPrefix", + "position": 17, + "type": "varchar(200)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "derivationSuffix", + "position": 18, + "type": "varchar(200)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "customInstructions", + "position": 19, + "type": "varchar(2500)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "spentBy", + "position": 20, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "sequenceNumber", + "position": 21, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "spendingDescription", + "position": 22, + "type": "varchar(2048)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "scriptLength", + "position": 23, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "scriptOffset", + "position": 24, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "lockingScript", + "position": 25, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "provenTxReqId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "provenTxId", + "position": 4, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "status", + "position": 5, + "type": "varchar(16)", + "nullable": "NO", + "defaultValue": "unknown", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "attempts", + "position": 6, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "notified", + "position": 7, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "txid", + "position": 8, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "batch", + "position": 9, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "history", + "position": 10, + "type": "longtext", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "notify", + "position": 11, + "type": "longtext", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "rawTx", + "position": 12, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "inputBEEF", + "position": 13, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "wasBroadcast", + "position": 14, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "rebroadcastAttempts", + "position": 15, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "provenTxId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "txid", + "position": 4, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "height", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "index", + "position": 6, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "merklePath", + "position": 7, + "type": "blob", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "rawTx", + "position": 8, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "blockHash", + "position": 9, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "merkleRoot", + "position": 10, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "settings", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "settings", + "name": "storageIdentityKey", + "position": 3, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "storageName", + "position": 4, + "type": "varchar(128)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "chain", + "position": 5, + "type": "varchar(10)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "dbtype", + "position": 6, + "type": "varchar(10)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "maxOutputScript", + "position": 7, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotUserId", + "position": 1, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotFieldName", + "position": 2, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotCertificateId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotMembership", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "transactionId", + "position": 1, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "requestId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "tableId", + "position": 3, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "rowId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "userId", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_guards", + "name": "proofId", + "position": 1, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_guards", + "name": "present", + "position": 2, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "tableId", + "position": 1, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "userId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "rowId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "refs", + "position": 4, + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "present", + "position": 5, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_profile_keys", + "name": "snapshotTableId", + "position": 1, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_profile_keys", + "name": "snapshotUserId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_profile_keys", + "name": "snapshotRowId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotTableId", + "position": 1, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotUserId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotLeftId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotRightId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotMembership", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "syncStateId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "storageIdentityKey", + "position": 5, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": "", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "storageName", + "position": 6, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "status", + "position": 7, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": "unknown", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "init", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "refNum", + "position": 9, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "syncMap", + "position": 10, + "type": "longtext", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "when", + "position": 11, + "type": "datetime", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "satoshis", + "position": 12, + "type": "bigint", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "errorLocal", + "position": 13, + "type": "longtext", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "errorOther", + "position": 14, + "type": "longtext", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "transactionId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "provenTxId", + "position": 5, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "status", + "position": 6, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "reference", + "position": 7, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "isOutgoing", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "satoshis", + "position": 9, + "type": "bigint", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "version", + "position": 10, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "lockTime", + "position": 11, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "description", + "position": 12, + "type": "varchar(2048)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "txid", + "position": 13, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "inputBEEF", + "position": 14, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "rawTx", + "position": 15, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryMode", + "position": 16, + "type": "varchar(16)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryValue", + "position": 17, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryDeadline", + "position": 18, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryState", + "position": 19, + "type": "varchar(24)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryAnchorTxid", + "position": 20, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryAnchorVout", + "position": 21, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReleasedAt", + "position": 22, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryObservedAt", + "position": 23, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimTxid", + "position": 24, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimRawTx", + "position": 25, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimDerivationPrefix", + "position": 26, + "type": "varchar(32)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimDerivationSuffix", + "position": 27, + "type": "varchar(32)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimSatoshis", + "position": 28, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "txLabelId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "label", + "position": 5, + "type": "varchar(300)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "isDeleted", + "position": 6, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "txLabelId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "transactionId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "isDeleted", + "position": 5, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "userId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "identityKey", + "position": 4, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "users", + "name": "activeStorage", + "position": 5, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + } + ] + }, + { + "sql": "SELECT TABLE_NAME tableName,INDEX_NAME name,SEQ_IN_INDEX position,COLUMN_NAME columnName,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,NULLABLE nullable,INDEX_TYPE type,IS_VISIBLE visible,SUBSTRING(EXPRESSION,1,16385) expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ORDER BY TABLE_NAME,INDEX_NAME,SEQ_IN_INDEX LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields", + "users", + "settings", + "snapshot_profile_keys", + "snapshot_relation_keys", + "snapshot_certificate_field_keys", + "snapshot_global_edges", + "snapshot_global_keys", + "snapshot_global_guards" + ], + "rows": [ + { + "tableName": "certificate_fields", + "name": "certificate_fields_certificateid_foreign", + "position": 1, + "columnName": "certificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_fieldname_certificateid_unique", + "position": 1, + "columnName": "fieldName", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_fieldname_certificateid_unique", + "position": 2, + "columnName": "certificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 1, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 2, + "columnName": "type", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 3, + "columnName": "certifier", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 4, + "columnName": "serialNumber", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "PRIMARY", + "position": 1, + "columnName": "certificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_index", + "position": 1, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_unique", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "commissions_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "PRIMARY", + "position": 1, + "columnName": "commissionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "output_baskets_name_userid_unique", + "position": 1, + "columnName": "name", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "output_baskets_name_userid_unique", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "output_baskets_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "PRIMARY", + "position": 1, + "columnName": "basketId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "output_tags_tag_userid_unique", + "position": 1, + "columnName": "tag", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "output_tags_tag_userid_unique", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "output_tags_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "PRIMARY", + "position": 1, + "columnName": "outputTagId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "position": 1, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "position": 2, + "columnName": "isDeleted", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "position": 3, + "columnName": "outputTagId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputid_index", + "position": 1, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_outputid_unique", + "position": 1, + "columnName": "outputTagId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_outputid_unique", + "position": 2, + "columnName": "outputId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 2, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 3, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 4, + "columnName": "spentBy", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 5, + "columnName": "satoshis", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 6, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_spentby", + "position": 1, + "columnName": "spentBy", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 2, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 3, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 4, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 2, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 3, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 4, + "columnName": "satoshis", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "position": 2, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "position": 3, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_basketid_foreign", + "position": 1, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_spendable_index", + "position": 1, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "position": 2, + "columnName": "vout", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "position": 3, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "PRIMARY", + "position": 1, + "columnName": "outputId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "PRIMARY", + "position": 1, + "columnName": "provenTxReqId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_batch_index", + "position": 1, + "columnName": "batch", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_status_index", + "position": 1, + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_txid_index", + "position": 1, + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_txid_unique", + "position": 1, + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_txs", + "name": "PRIMARY", + "position": 1, + "columnName": "provenTxId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_txs", + "name": "proven_txs_blockhash_index", + "position": 1, + "columnName": "blockHash", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_txs", + "name": "proven_txs_txid_unique", + "position": 1, + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "snapshotFieldName", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "snapshotCertificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_lookup", + "position": 1, + "columnName": "snapshotFieldName", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_lookup", + "position": 2, + "columnName": "snapshotCertificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_lookup", + "position": 3, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_parent", + "position": 1, + "columnName": "snapshotCertificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_parent", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_parent", + "position": 3, + "columnName": "snapshotFieldName", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 2, + "columnName": "requestId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 3, + "columnName": "tableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 4, + "columnName": "rowId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_request", + "position": 1, + "columnName": "requestId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_request", + "position": 2, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_guards", + "name": "PRIMARY", + "position": 1, + "columnName": "proofId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "tableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "rowId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 1, + "columnName": "tableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 2, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 3, + "columnName": "present", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 4, + "columnName": "rowId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_target", + "position": 1, + "columnName": "tableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_target", + "position": 2, + "columnName": "rowId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_target", + "position": 3, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_profile_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_profile_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_profile_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "snapshotRowId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "snapshotLeftId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 4, + "columnName": "snapshotRightId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 2, + "columnName": "snapshotLeftId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 3, + "columnName": "snapshotRightId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 4, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 3, + "columnName": "snapshotRightId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 4, + "columnName": "snapshotLeftId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "PRIMARY", + "position": 1, + "columnName": "syncStateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_refnum_index", + "position": 1, + "columnName": "refNum", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_refnum_unique", + "position": 1, + "columnName": "refNum", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_status_index", + "position": 1, + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_expiry", + "position": 1, + "columnName": "noSendExpiryState", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_expiry", + "position": 2, + "columnName": "noSendExpiryDeadline", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_reclaim", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_reclaim", + "position": 2, + "columnName": "noSendExpiryReclaimTxid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_proven_tx", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_proven_tx", + "position": 2, + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_txid", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_txid", + "position": 2, + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "PRIMARY", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_reference_unique", + "position": 1, + "columnName": "reference", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_status_index", + "position": 1, + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_txid_index", + "position": 1, + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "PRIMARY", + "position": 1, + "columnName": "txLabelId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "tx_labels_label_userid_unique", + "position": 1, + "columnName": "label", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "tx_labels_label_userid_unique", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "tx_labels_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "idx_tx_labels_map_tx_deleted", + "position": 1, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "idx_tx_labels_map_tx_deleted", + "position": 2, + "columnName": "isDeleted", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_transactionid_index", + "position": 1, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_transactionid_unique", + "position": 1, + "columnName": "txLabelId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_transactionid_unique", + "position": 2, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "users", + "name": "PRIMARY", + "position": 1, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "users", + "name": "users_identitykey_unique", + "position": 1, + "columnName": "identityKey", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT TABLE_NAME tableName,CONSTRAINT_NAME name,ORDINAL_POSITION position,COLUMN_NAME columnName,REFERENCED_TABLE_SCHEMA foreignSchema,REFERENCED_TABLE_NAME foreignTable,REFERENCED_COLUMN_NAME foreignColumn FROM information_schema.KEY_COLUMN_USAGE WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) AND REFERENCED_TABLE_NAME IS NOT NULL ORDER BY TABLE_NAME,CONSTRAINT_NAME,ORDINAL_POSITION LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields", + "users", + "settings", + "snapshot_profile_keys", + "snapshot_relation_keys", + "snapshot_certificate_field_keys", + "snapshot_global_edges", + "snapshot_global_keys", + "snapshot_global_guards" + ], + "rows": [ + { + "tableName": "certificate_fields", + "name": "certificate_fields_certificateid_foreign", + "position": 1, + "columnName": "certificateId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "certificates", + "foreignColumn": "certificateId" + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "certificates", + "name": "certificates_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_foreign", + "position": 1, + "columnName": "transactionId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "commissions", + "name": "commissions_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "output_baskets", + "name": "output_baskets_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "output_tags", + "name": "output_tags_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputid_foreign", + "position": 1, + "columnName": "outputId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "outputs", + "foreignColumn": "outputId" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_foreign", + "position": 1, + "columnName": "outputTagId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "output_tags", + "foreignColumn": "outputTagId" + }, + { + "tableName": "outputs", + "name": "outputs_basketid_foreign", + "position": 1, + "columnName": "basketId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "output_baskets", + "foreignColumn": "basketId" + }, + { + "tableName": "outputs", + "name": "outputs_spentby_foreign", + "position": 1, + "columnName": "spentBy", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_foreign", + "position": 1, + "columnName": "transactionId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "outputs", + "name": "outputs_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "proven_txs", + "foreignColumn": "provenTxId" + }, + { + "tableName": "sync_states", + "name": "sync_states_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "transactions", + "name": "transactions_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "proven_txs", + "foreignColumn": "provenTxId" + }, + { + "tableName": "transactions", + "name": "transactions_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "tx_labels", + "name": "tx_labels_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_transactionid_foreign", + "position": 1, + "columnName": "transactionId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_foreign", + "position": 1, + "columnName": "txLabelId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "tx_labels", + "foreignColumn": "txLabelId" + } + ] + }, + { + "sql": "SELECT TABLE_NAME tableName,CONSTRAINT_NAME name,MATCH_OPTION matchOption,UPDATE_RULE updateRule,DELETE_RULE deleteRule FROM information_schema.REFERENTIAL_CONSTRAINTS WHERE CONSTRAINT_SCHEMA=DATABASE() AND TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ORDER BY TABLE_NAME,CONSTRAINT_NAME LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields", + "users", + "settings", + "snapshot_profile_keys", + "snapshot_relation_keys", + "snapshot_certificate_field_keys", + "snapshot_global_edges", + "snapshot_global_keys", + "snapshot_global_guards" + ], + "rows": [ + { + "tableName": "certificate_fields", + "name": "certificate_fields_certificateid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "certificates", + "name": "certificates_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "commissions", + "name": "commissions_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_baskets", + "name": "output_baskets_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_tags", + "name": "output_tags_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_basketid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_spentby_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_proventxid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "sync_states", + "name": "sync_states_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "transactions", + "name": "transactions_proventxid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "transactions", + "name": "transactions_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "tx_labels", + "name": "tx_labels_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_transactionid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + } + ] + }, + { + "sql": "SELECT t.TABLE_NAME tableName,t.CONSTRAINT_NAME name,t.ENFORCED enforced,SUBSTRING(c.CHECK_CLAUSE,1,16385) clause FROM information_schema.TABLE_CONSTRAINTS t JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) AND t.CONSTRAINT_TYPE='CHECK' ORDER BY t.TABLE_NAME,t.CONSTRAINT_NAME LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields", + "users", + "settings", + "snapshot_profile_keys", + "snapshot_relation_keys", + "snapshot_certificate_field_keys", + "snapshot_global_edges", + "snapshot_global_keys", + "snapshot_global_guards" + ], + "rows": [] + }, + { + "sql": "SELECT EVENT_OBJECT_TABLE tableName,TRIGGER_NAME name,EVENT_MANIPULATION event,ACTION_TIMING timing,ACTION_ORDER actionOrder,DEFINER definer,SUBSTRING(ACTION_STATEMENT,1,16385) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION connectionCollation,DATABASE_COLLATION databaseCollation FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) AND TRIGGER_NAME NOT IN (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?) ORDER BY EVENT_OBJECT_TABLE,TRIGGER_NAME LIMIT 513", + "bindings": [ + "transactions", + "outputs", + "certificates", + "tx_labels", + "output_baskets", + "output_tags", + "commissions", + "sync_states", + "proven_txs", + "proven_tx_reqs", + "tx_labels_map", + "output_tags_map", + "certificate_fields", + "users", + "settings", + "snapshot_profile_keys", + "snapshot_relation_keys", + "snapshot_certificate_field_keys", + "snapshot_global_edges", + "snapshot_global_keys", + "snapshot_global_guards", + "snapshot_journal_scope_0_INSERT", + "snapshot_journal_scope_0_UPDATE", + "snapshot_journal_scope_0_DELETE", + "snapshot_journal_scope_1_INSERT", + "snapshot_journal_scope_1_UPDATE", + "snapshot_journal_scope_1_DELETE", + "snapshot_journal_scope_2_INSERT", + "snapshot_journal_scope_2_UPDATE", + "snapshot_journal_scope_2_DELETE", + "snapshot_journal_scope_3_INSERT", + "snapshot_journal_scope_3_UPDATE", + "snapshot_journal_scope_3_DELETE", + "snapshot_journal_physical_0_INSERT", + "snapshot_journal_physical_0_UPDATE", + "snapshot_journal_physical_0_DELETE", + "snapshot_journal_physical_1_INSERT", + "snapshot_journal_physical_1_UPDATE", + "snapshot_journal_physical_1_DELETE", + "snapshot_journal_physical_2_INSERT", + "snapshot_journal_physical_2_UPDATE", + "snapshot_journal_physical_2_DELETE", + "snapshot_journal_physical_3_INSERT", + "snapshot_journal_physical_3_UPDATE", + "snapshot_journal_physical_3_DELETE", + "snapshot_journal_physical_4_INSERT", + "snapshot_journal_physical_4_UPDATE", + "snapshot_journal_physical_4_DELETE", + "snapshot_journal_physical_5_INSERT", + "snapshot_journal_physical_5_UPDATE", + "snapshot_journal_physical_5_DELETE", + "snapshot_journal_physical_6_INSERT", + "snapshot_journal_physical_6_UPDATE", + "snapshot_journal_physical_6_DELETE", + "snapshot_journal_physical_7_INSERT", + "snapshot_journal_physical_7_UPDATE", + "snapshot_journal_physical_7_DELETE", + "snapshot_journal_physical_8_INSERT", + "snapshot_journal_physical_8_UPDATE", + "snapshot_journal_physical_8_DELETE", + "snapshot_journal_physical_9_INSERT", + "snapshot_journal_physical_9_UPDATE", + "snapshot_journal_physical_9_DELETE", + "snapshot_journal_physical_10_INSERT", + "snapshot_journal_physical_10_UPDATE", + "snapshot_journal_physical_10_DELETE", + "snapshot_journal_physical_11_INSERT", + "snapshot_journal_physical_11_UPDATE", + "snapshot_journal_physical_11_DELETE", + "snapshot_journal_physical_12_INSERT", + "snapshot_journal_physical_12_UPDATE", + "snapshot_journal_physical_12_DELETE" + ], + "rows": [ + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_profile_2_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_profile_2_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_profile_2_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "commissions", + "name": "snapshot_profile_6_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "commissions", + "name": "snapshot_profile_6_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "commissions", + "name": "snapshot_profile_6_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.commissionId <=> NEW.commissionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_baskets", + "name": "snapshot_profile_4_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_baskets", + "name": "snapshot_profile_4_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_baskets", + "name": "snapshot_profile_4_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.basketId <=> NEW.basketId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_profile_5_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_profile_5_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_profile_5_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputTagId <=> NEW.outputTagId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_profile_1_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_profile_1_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_profile_1_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_edge_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_global_keys SET refs=refs-1 WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId; DELETE FROM snapshot_global_keys WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId AND refs=0; DELETE FROM snapshot_global_guards WHERE proofId=OLD.rowId AND OLD.tableId=1 AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.rowId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_edge_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_global_keys (tableId,userId,rowId,refs,present) VALUES (NEW.tableId,NEW.userId,NEW.rowId,1,CASE WHEN NEW.tableId=0 THEN 1 ELSE (SELECT present FROM snapshot_global_guards WHERE proofId=NEW.rowId FOR SHARE) END) ON DUPLICATE KEY UPDATE refs=snapshot_global_keys.refs+1; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "sync_states", + "name": "snapshot_profile_7_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "sync_states", + "name": "snapshot_profile_7_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "sync_states", + "name": "snapshot_profile_7_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.syncStateId <=> NEW.syncStateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 3, + "definer": "root@%", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 3, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 3, + "definer": "root@%", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_profile_0_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_profile_0_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_profile_0_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_profile_3_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_profile_3_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_profile_3_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txLabelId <=> NEW.txLabelId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "users", + "name": "snapshot_sync_primary_change", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.activeStorage <=> NEW.activeStorage) THEN\n INSERT INTO snapshot_sync_primary_epochs (userId, epoch) VALUES (NEW.userId, 1)\n ON DUPLICATE KEY UPDATE epoch = epoch + 1;\n END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + } + ] + }, + { + "sql": "SELECT @@sql_mode sqlMode,@@character_set_client charset,@@collation_connection collation,@@collation_database databaseCollation,CURRENT_USER() definer", + "bindings": [], + "rows": [ + { + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["transactions"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "transactionId" + }, + { + "name": "userId" + }, + { + "name": "provenTxId" + }, + { + "name": "status" + }, + { + "name": "reference" + }, + { + "name": "isOutgoing" + }, + { + "name": "satoshis" + }, + { + "name": "version" + }, + { + "name": "lockTime" + }, + { + "name": "description" + }, + { + "name": "txid" + }, + { + "name": "inputBEEF" + }, + { + "name": "rawTx" + }, + { + "name": "noSendExpiryMode" + }, + { + "name": "noSendExpiryValue" + }, + { + "name": "noSendExpiryDeadline" + }, + { + "name": "noSendExpiryState" + }, + { + "name": "noSendExpiryAnchorTxid" + }, + { + "name": "noSendExpiryAnchorVout" + }, + { + "name": "noSendExpiryReleasedAt" + }, + { + "name": "noSendExpiryObservedAt" + }, + { + "name": "noSendExpiryReclaimTxid" + }, + { + "name": "noSendExpiryReclaimRawTx" + }, + { + "name": "noSendExpiryReclaimDerivationPrefix" + }, + { + "name": "noSendExpiryReclaimDerivationSuffix" + }, + { + "name": "noSendExpiryReclaimSatoshis" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["outputs"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "outputId" + }, + { + "name": "userId" + }, + { + "name": "transactionId" + }, + { + "name": "basketId" + }, + { + "name": "spendable" + }, + { + "name": "change" + }, + { + "name": "vout" + }, + { + "name": "satoshis" + }, + { + "name": "providedBy" + }, + { + "name": "purpose" + }, + { + "name": "type" + }, + { + "name": "outputDescription" + }, + { + "name": "txid" + }, + { + "name": "senderIdentityKey" + }, + { + "name": "derivationPrefix" + }, + { + "name": "derivationSuffix" + }, + { + "name": "customInstructions" + }, + { + "name": "spentBy" + }, + { + "name": "sequenceNumber" + }, + { + "name": "spendingDescription" + }, + { + "name": "scriptLength" + }, + { + "name": "scriptOffset" + }, + { + "name": "lockingScript" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["certificates"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "certificateId" + }, + { + "name": "userId" + }, + { + "name": "serialNumber" + }, + { + "name": "type" + }, + { + "name": "certifier" + }, + { + "name": "subject" + }, + { + "name": "verifier" + }, + { + "name": "revocationOutpoint" + }, + { + "name": "signature" + }, + { + "name": "isDeleted" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["tx_labels"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "txLabelId" + }, + { + "name": "userId" + }, + { + "name": "label" + }, + { + "name": "isDeleted" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["output_baskets"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "basketId" + }, + { + "name": "userId" + }, + { + "name": "name" + }, + { + "name": "numberOfDesiredUTXOs" + }, + { + "name": "minimumDesiredUTXOValue" + }, + { + "name": "isDeleted" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["output_tags"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "outputTagId" + }, + { + "name": "userId" + }, + { + "name": "tag" + }, + { + "name": "isDeleted" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["commissions"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "commissionId" + }, + { + "name": "userId" + }, + { + "name": "transactionId" + }, + { + "name": "satoshis" + }, + { + "name": "keyOffset" + }, + { + "name": "isRedeemed" + }, + { + "name": "lockingScript" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["sync_states"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "syncStateId" + }, + { + "name": "userId" + }, + { + "name": "storageIdentityKey" + }, + { + "name": "storageName" + }, + { + "name": "status" + }, + { + "name": "init" + }, + { + "name": "refNum" + }, + { + "name": "syncMap" + }, + { + "name": "when" + }, + { + "name": "satoshis" + }, + { + "name": "errorLocal" + }, + { + "name": "errorOther" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["proven_txs"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "provenTxId" + }, + { + "name": "txid" + }, + { + "name": "height" + }, + { + "name": "index" + }, + { + "name": "merklePath" + }, + { + "name": "rawTx" + }, + { + "name": "blockHash" + }, + { + "name": "merkleRoot" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["proven_tx_reqs"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "provenTxReqId" + }, + { + "name": "provenTxId" + }, + { + "name": "status" + }, + { + "name": "attempts" + }, + { + "name": "notified" + }, + { + "name": "txid" + }, + { + "name": "batch" + }, + { + "name": "history" + }, + { + "name": "notify" + }, + { + "name": "rawTx" + }, + { + "name": "inputBEEF" + }, + { + "name": "wasBroadcast" + }, + { + "name": "rebroadcastAttempts" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["tx_labels_map"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "txLabelId" + }, + { + "name": "transactionId" + }, + { + "name": "isDeleted" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["output_tags_map"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "outputTagId" + }, + { + "name": "outputId" + }, + { + "name": "isDeleted" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION", + "bindings": ["certificate_fields"], + "rows": [ + { + "name": "created_at" + }, + { + "name": "updated_at" + }, + { + "name": "userId" + }, + { + "name": "certificateId" + }, + { + "name": "fieldName" + }, + { + "name": "fieldValue" + }, + { + "name": "masterKey" + } + ] + }, + { + "sql": "SELECT TABLE_NAME name,TABLE_TYPE type FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LOWER(LEFT(TABLE_NAME,17))='snapshot_journal_' ORDER BY TABLE_NAME LIMIT 130", + "bindings": [], + "rows": [ + { + "name": "snapshot_journal_bootstrap", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_clock", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_events", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_generation", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_invalid", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_physical", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_receipts", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_retention", + "type": "BASE TABLE" + }, + { + "name": "snapshot_journal_scope", + "type": "BASE TABLE" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,'TRIGGER' type FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND LOWER(LEFT(TRIGGER_NAME,17))='snapshot_journal_' ORDER BY TRIGGER_NAME LIMIT 130", + "bindings": [], + "rows": [ + { + "name": "snapshot_journal_physical_0_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_0_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_0_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_10_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_10_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_10_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_11_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_11_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_11_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_12_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_12_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_12_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_1_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_1_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_1_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_2_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_2_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_2_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_3_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_3_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_3_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_4_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_4_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_4_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_5_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_5_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_5_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_6_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_6_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_6_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_7_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_7_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_7_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_8_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_8_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_8_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_9_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_9_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_physical_9_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_0_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_0_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_0_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_1_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_1_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_1_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_2_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_2_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_2_UPDATE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_3_DELETE", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_3_INSERT", + "type": "TRIGGER" + }, + { + "name": "snapshot_journal_scope_3_UPDATE", + "type": "TRIGGER" + } + ] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_generation"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "ascii_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_generation"], + "rows": [ + { + "name": "id", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "version", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "epoch", + "type": "varchar(36)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "source", + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "plan", + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "ceiling", + "type": "varchar(19)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "nextObject", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "complete", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 2", + "bindings": ["snapshot_journal_generation"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "id", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES" + } + ] + }, + { + "sql": "SELECT CONSTRAINT_NAME name,CONSTRAINT_TYPE type,ENFORCED enforced FROM information_schema.TABLE_CONSTRAINTS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? LIMIT 2", + "bindings": ["snapshot_journal_generation"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_generation"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_clock"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_clock"], + "rows": [ + { + "name": "id", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "ceiling", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_clock"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "id", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_clock"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + }, + { + "name": "snapshot_journal_clock_chk_1", + "type": "CHECK", + "enforced": "YES", + "clause": "(`id` = 1)" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_clock"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_clock"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_clock"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_events"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_events"], + "rows": [ + { + "name": "revision", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_events"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "revision", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_events"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_events"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_events"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_events"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_invalid"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_invalid"], + "rows": [ + { + "name": "id", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "reason", + "type": "varchar(32)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_invalid"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "id", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_invalid"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + }, + { + "name": "snapshot_journal_invalid_chk_1", + "type": "CHECK", + "enforced": "YES", + "clause": "(`id` = 1)" + }, + { + "name": "snapshot_journal_invalid_chk_2", + "type": "CHECK", + "enforced": "YES", + "clause": "(`reason` in (_utf8mb4\\'capacity-exhausted\\',_utf8mb4\\'revision-exhausted\\',_utf8mb4\\'key-out-of-range\\'))" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_invalid"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_invalid"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_invalid"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_physical"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_physical"], + "rows": [ + { + "name": "tableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "id1", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "id2", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "exactText", + "type": "varbinary(400)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "revision", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "generation", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "present", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_physical"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "tableId", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "PRIMARY", + "columnName": "id1", + "position": 2, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "PRIMARY", + "columnName": "id2", + "position": 3, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "PRIMARY", + "columnName": "exactText", + "position": 4, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_physical_page", + "columnName": "tableId", + "position": 1, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_physical_page", + "columnName": "revision", + "position": 2, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_physical_page", + "columnName": "id1", + "position": 3, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_physical_page", + "columnName": "id2", + "position": 4, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_physical_page", + "columnName": "exactText", + "position": 5, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_physical"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_physical"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_physical"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_physical"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_scope"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_scope"], + "rows": [ + { + "name": "tableId", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "userId", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "id1", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "id2", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "exactText", + "type": "varbinary(400)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "revision", + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "present", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_scope"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "tableId", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "PRIMARY", + "columnName": "userId", + "position": 2, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "PRIMARY", + "columnName": "id1", + "position": 3, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "PRIMARY", + "columnName": "id2", + "position": 4, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "PRIMARY", + "columnName": "exactText", + "position": 5, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_scope_page", + "columnName": "userId", + "position": 1, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_scope_page", + "columnName": "tableId", + "position": 2, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_scope_page", + "columnName": "revision", + "position": 3, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_scope_page", + "columnName": "id1", + "position": 4, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_scope_page", + "columnName": "id2", + "position": 5, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_scope_page", + "columnName": "exactText", + "position": 6, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_scope"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_scope"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_scope"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_scope"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_bootstrap"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_bootstrap"], + "rows": [ + { + "name": "id", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "stream", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "cursor", + "type": "text", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_bin", + "expression": "" + }, + { + "name": "rowLimit", + "type": "int", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "rowsUsed", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_bootstrap"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "id", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_bootstrap"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + }, + { + "name": "snapshot_journal_bootstrap_chk_1", + "type": "CHECK", + "enforced": "YES", + "clause": "(`id` = 1)" + }, + { + "name": "snapshot_journal_bootstrap_chk_2", + "type": "CHECK", + "enforced": "YES", + "clause": "(`stream` between 0 and 17)" + }, + { + "name": "snapshot_journal_bootstrap_chk_3", + "type": "CHECK", + "enforced": "YES", + "clause": "((`rowLimit` is null) or (`rowLimit` between 0 and 2147483647))" + }, + { + "name": "snapshot_journal_bootstrap_chk_4", + "type": "CHECK", + "enforced": "YES", + "clause": "(`rowsUsed` between 0 and 2147483647)" + }, + { + "name": "snapshot_journal_bootstrap_chk_5", + "type": "CHECK", + "enforced": "YES", + "clause": "((`rowLimit` is null) or (`rowsUsed` <= `rowLimit`))" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_bootstrap"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_bootstrap"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_bootstrap"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_retention"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "ascii_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_retention"], + "rows": [ + { + "name": "id", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "floor", + "type": "varchar(19)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "receiptLimit", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "receiptLifetimeMs", + "type": "bigint", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_retention"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "id", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_retention"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + }, + { + "name": "snapshot_journal_retention_chk_1", + "type": "CHECK", + "enforced": "YES", + "clause": "(`id` = 1)" + }, + { + "name": "snapshot_journal_retention_chk_2", + "type": "CHECK", + "enforced": "YES", + "clause": "(`receiptLimit` between 1 and 128)" + }, + { + "name": "snapshot_journal_retention_chk_3", + "type": "CHECK", + "enforced": "YES", + "clause": "(`receiptLifetimeMs` between 1 and 2592000000)" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_retention"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_retention"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_retention"], + "rows": [] + }, + { + "sql": "SELECT ENGINE engine,TABLE_TYPE type,TABLE_COLLATION collation,ROW_FORMAT rowFormat,CREATE_OPTIONS options,TABLE_COMMENT comment FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=?", + "bindings": ["snapshot_journal_receipts"], + "rows": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "ascii_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC", + "comment": "snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066" + } + ] + }, + { + "sql": "SELECT COLUMN_NAME name,COLUMN_TYPE type,IS_NULLABLE nullable,COLUMN_DEFAULT defaultValue,EXTRA extra,CHARACTER_SET_NAME charset,COLLATION_NAME collation,GENERATION_EXPRESSION expression FROM information_schema.COLUMNS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY ORDINAL_POSITION LIMIT 9", + "bindings": ["snapshot_journal_receipts"], + "rows": [ + { + "name": "requestId", + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "binding", + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "highWater", + "type": "varchar(19)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "floor", + "type": "varchar(19)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "expiresAt", + "type": "bigint", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME columnName,SEQ_IN_INDEX position,NON_UNIQUE nonUnique,COLLATION direction,SUB_PART prefix,INDEX_TYPE type,IS_VISIBLE visible,EXPRESSION expression FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX LIMIT 16", + "bindings": ["snapshot_journal_receipts"], + "rows": [ + { + "name": "PRIMARY", + "columnName": "requestId", + "position": 1, + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_receipts_expiry", + "columnName": "expiresAt", + "position": 1, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "name": "snapshot_journal_receipts_expiry", + "columnName": "requestId", + "position": 2, + "nonUnique": 1, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES", + "expression": null + } + ] + }, + { + "sql": "SELECT t.CONSTRAINT_NAME name,t.CONSTRAINT_TYPE type,t.ENFORCED enforced,c.CHECK_CLAUSE clause FROM information_schema.TABLE_CONSTRAINTS t LEFT JOIN information_schema.CHECK_CONSTRAINTS c ON c.CONSTRAINT_SCHEMA=t.CONSTRAINT_SCHEMA AND c.CONSTRAINT_NAME=t.CONSTRAINT_NAME WHERE t.TABLE_SCHEMA=DATABASE() AND t.TABLE_NAME=? ORDER BY t.CONSTRAINT_NAME LIMIT 8", + "bindings": ["snapshot_journal_receipts"], + "rows": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES", + "clause": null + }, + { + "name": "snapshot_journal_receipts_chk_1", + "type": "CHECK", + "enforced": "YES", + "clause": "(`expiresAt` between 1 and 9007199254740991)" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND EVENT_OBJECT_TABLE=? LIMIT 1", + "bindings": ["snapshot_journal_receipts"], + "rows": [] + }, + { + "sql": "SELECT PARTITION_NAME FROM information_schema.PARTITIONS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? AND PARTITION_NAME IS NOT NULL LIMIT 1", + "bindings": ["snapshot_journal_receipts"], + "rows": [] + }, + { + "sql": "SELECT CONSTRAINT_NAME FROM information_schema.KEY_COLUMN_USAGE WHERE REFERENCED_TABLE_SCHEMA=DATABASE() AND REFERENCED_TABLE_NAME=? LIMIT 1", + "bindings": ["snapshot_journal_receipts"], + "rows": [] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1444, "snapshot_journal_scope_0_INSERT"], + "rows": [ + { + "name": "snapshot_journal_scope_0_INSERT", + "table": "snapshot_profile_keys", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotRowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId,NEW.snapshotUserId,NEW.snapshotRowId,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1444, "snapshot_journal_scope_0_DELETE"], + "rows": [ + { + "name": "snapshot_journal_scope_0_DELETE", + "table": "snapshot_profile_keys", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.snapshotRowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT OLD.snapshotTableId,OLD.snapshotUserId,OLD.snapshotRowId,0,CAST('' AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1674, "snapshot_journal_scope_0_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_scope_0_UPDATE", + "table": "snapshot_profile_keys", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (OLD.snapshotTableId <=> NEW.snapshotTableId) OR NOT (OLD.snapshotUserId <=> NEW.snapshotUserId) OR NOT (OLD.snapshotRowId <=> NEW.snapshotRowId) OR NOT (0 <=> 0) OR NOT (CAST('' AS BINARY) <=> CAST('' AS BINARY)) OR NOT (1 <=> 1) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotRowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId,NEW.snapshotUserId,NEW.snapshotRowId,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1511, "snapshot_journal_scope_1_INSERT"], + "rows": [ + { + "name": "snapshot_journal_scope_1_INSERT", + "table": "snapshot_relation_keys", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotLeftId BETWEEN 1 AND 9007199254740991) AND (NEW.snapshotRightId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId+10,NEW.snapshotUserId,NEW.snapshotLeftId,NEW.snapshotRightId,CAST('' AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1485, "snapshot_journal_scope_1_DELETE"], + "rows": [ + { + "name": "snapshot_journal_scope_1_DELETE", + "table": "snapshot_relation_keys", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.snapshotLeftId BETWEEN 1 AND 9007199254740991) AND (OLD.snapshotRightId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT OLD.snapshotTableId+10,OLD.snapshotUserId,OLD.snapshotLeftId,OLD.snapshotRightId,CAST('' AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1837, "snapshot_journal_scope_1_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_scope_1_UPDATE", + "table": "snapshot_relation_keys", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (OLD.snapshotTableId+10 <=> NEW.snapshotTableId+10) OR NOT (OLD.snapshotUserId <=> NEW.snapshotUserId) OR NOT (OLD.snapshotLeftId <=> NEW.snapshotLeftId) OR NOT (OLD.snapshotRightId <=> NEW.snapshotRightId) OR NOT (CAST('' AS BINARY) <=> CAST('' AS BINARY)) OR NOT ((OLD.snapshotMembership<>0) <=> (NEW.snapshotMembership<>0)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotLeftId BETWEEN 1 AND 9007199254740991) AND (NEW.snapshotRightId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT NEW.snapshotTableId+10,NEW.snapshotUserId,NEW.snapshotLeftId,NEW.snapshotRightId,CAST('' AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1507, "snapshot_journal_scope_2_INSERT"], + "rows": [ + { + "name": "snapshot_journal_scope_2_INSERT", + "table": "snapshot_certificate_field_keys", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotCertificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.snapshotFieldName AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,NEW.snapshotUserId,NEW.snapshotCertificateId,0,CAST(NEW.snapshotFieldName AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1481, "snapshot_journal_scope_2_DELETE"], + "rows": [ + { + "name": "snapshot_journal_scope_2_DELETE", + "table": "snapshot_certificate_field_keys", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.snapshotCertificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(OLD.snapshotFieldName AS BINARY))<=400) AND (OLD.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,OLD.snapshotUserId,OLD.snapshotCertificateId,0,CAST(OLD.snapshotFieldName AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1809, "snapshot_journal_scope_2_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_scope_2_UPDATE", + "table": "snapshot_certificate_field_keys", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (12 <=> 12) OR NOT (OLD.snapshotUserId <=> NEW.snapshotUserId) OR NOT (OLD.snapshotCertificateId <=> NEW.snapshotCertificateId) OR NOT (0 <=> 0) OR NOT (CAST(OLD.snapshotFieldName AS BINARY) <=> CAST(NEW.snapshotFieldName AS BINARY)) OR NOT ((OLD.snapshotMembership<>0) <=> (NEW.snapshotMembership<>0)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.snapshotCertificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.snapshotFieldName AS BINARY))<=400) AND (NEW.snapshotUserId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,NEW.snapshotUserId,NEW.snapshotCertificateId,0,CAST(NEW.snapshotFieldName AS BINARY),journalRevision,(NEW.snapshotMembership<>0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1444, "snapshot_journal_scope_3_INSERT"], + "rows": [ + { + "name": "snapshot_journal_scope_3_INSERT", + "table": "snapshot_global_keys", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.rowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT CASE NEW.tableId WHEN 0 THEN 9 ELSE 8 END,NEW.userId,NEW.rowId,0,CAST('' AS BINARY),journalRevision,NEW.present ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1434, "snapshot_journal_scope_3_DELETE"], + "rows": [ + { + "name": "snapshot_journal_scope_3_DELETE", + "table": "snapshot_global_keys", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.rowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT CASE OLD.tableId WHEN 0 THEN 9 ELSE 8 END,OLD.userId,OLD.rowId,0,CAST('' AS BINARY),journalRevision,0 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1706, "snapshot_journal_scope_3_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_scope_3_UPDATE", + "table": "snapshot_global_keys", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CASE OLD.tableId WHEN 0 THEN 9 ELSE 8 END <=> CASE NEW.tableId WHEN 0 THEN 9 ELSE 8 END) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.rowId <=> NEW.rowId) OR NOT (0 <=> 0) OR NOT (CAST('' AS BINARY) <=> CAST('' AS BINARY)) OR NOT (OLD.present <=> NEW.present) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.rowId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT CASE NEW.tableId WHEN 0 THEN 9 ELSE 8 END,NEW.userId,NEW.rowId,0,CAST('' AS BINARY),journalRevision,NEW.present ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1737, "snapshot_journal_physical_0_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_0_INSERT", + "table": "transactions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 0,NEW.userId,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [5134, "snapshot_journal_physical_0_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_0_UPDATE", + "table": "transactions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`provenTxId` AS BINARY) <=> CAST(NEW.`provenTxId` AS BINARY)) OR NOT (CAST(OLD.`status` AS BINARY) <=> CAST(NEW.`status` AS BINARY)) OR NOT (CAST(OLD.`reference` AS BINARY) <=> CAST(NEW.`reference` AS BINARY)) OR NOT (CAST(OLD.`isOutgoing` AS BINARY) <=> CAST(NEW.`isOutgoing` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`version` AS BINARY) <=> CAST(NEW.`version` AS BINARY)) OR NOT (CAST(OLD.`lockTime` AS BINARY) <=> CAST(NEW.`lockTime` AS BINARY)) OR NOT (CAST(OLD.`description` AS BINARY) <=> CAST(NEW.`description` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`inputBEEF` AS BINARY) <=> CAST(NEW.`inputBEEF` AS BINARY)) OR NOT (CAST(OLD.`rawTx` AS BINARY) <=> CAST(NEW.`rawTx` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryMode` AS BINARY) <=> CAST(NEW.`noSendExpiryMode` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryValue` AS BINARY) <=> CAST(NEW.`noSendExpiryValue` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryDeadline` AS BINARY) <=> CAST(NEW.`noSendExpiryDeadline` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryState` AS BINARY) <=> CAST(NEW.`noSendExpiryState` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryAnchorTxid` AS BINARY) <=> CAST(NEW.`noSendExpiryAnchorTxid` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryAnchorVout` AS BINARY) <=> CAST(NEW.`noSendExpiryAnchorVout` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReleasedAt` AS BINARY) <=> CAST(NEW.`noSendExpiryReleasedAt` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryObservedAt` AS BINARY) <=> CAST(NEW.`noSendExpiryObservedAt` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimTxid` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimTxid` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimRawTx` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimRawTx` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimDerivationPrefix` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimDerivationPrefix` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimDerivationSuffix` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimDerivationSuffix` AS BINARY)) OR NOT (CAST(OLD.`noSendExpiryReclaimSatoshis` AS BINARY) <=> CAST(NEW.`noSendExpiryReclaimSatoshis` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`transactionId` <=> NEW.`transactionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,OLD.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`transactionId` <=> NEW.`transactionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 0,NEW.userId,NEW.`transactionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1498, "snapshot_journal_physical_0_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_0_DELETE", + "table": "transactions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`transactionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(0,OLD.`transactionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1722, "snapshot_journal_physical_1_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_1_INSERT", + "table": "outputs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 1,NEW.userId,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [4592, "snapshot_journal_physical_1_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_1_UPDATE", + "table": "outputs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`outputId` AS BINARY) <=> CAST(NEW.`outputId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`basketId` AS BINARY) <=> CAST(NEW.`basketId` AS BINARY)) OR NOT (CAST(OLD.`spendable` AS BINARY) <=> CAST(NEW.`spendable` AS BINARY)) OR NOT (CAST(OLD.`change` AS BINARY) <=> CAST(NEW.`change` AS BINARY)) OR NOT (CAST(OLD.`vout` AS BINARY) <=> CAST(NEW.`vout` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`providedBy` AS BINARY) <=> CAST(NEW.`providedBy` AS BINARY)) OR NOT (CAST(OLD.`purpose` AS BINARY) <=> CAST(NEW.`purpose` AS BINARY)) OR NOT (CAST(OLD.`type` AS BINARY) <=> CAST(NEW.`type` AS BINARY)) OR NOT (CAST(OLD.`outputDescription` AS BINARY) <=> CAST(NEW.`outputDescription` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`senderIdentityKey` AS BINARY) <=> CAST(NEW.`senderIdentityKey` AS BINARY)) OR NOT (CAST(OLD.`derivationPrefix` AS BINARY) <=> CAST(NEW.`derivationPrefix` AS BINARY)) OR NOT (CAST(OLD.`derivationSuffix` AS BINARY) <=> CAST(NEW.`derivationSuffix` AS BINARY)) OR NOT (CAST(OLD.`customInstructions` AS BINARY) <=> CAST(NEW.`customInstructions` AS BINARY)) OR NOT (CAST(OLD.`spentBy` AS BINARY) <=> CAST(NEW.`spentBy` AS BINARY)) OR NOT (CAST(OLD.`sequenceNumber` AS BINARY) <=> CAST(NEW.`sequenceNumber` AS BINARY)) OR NOT (CAST(OLD.`spendingDescription` AS BINARY) <=> CAST(NEW.`spendingDescription` AS BINARY)) OR NOT (CAST(OLD.`scriptLength` AS BINARY) <=> CAST(NEW.`scriptLength` AS BINARY)) OR NOT (CAST(OLD.`scriptOffset` AS BINARY) <=> CAST(NEW.`scriptOffset` AS BINARY)) OR NOT (CAST(OLD.`lockingScript` AS BINARY) <=> CAST(NEW.`lockingScript` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`outputId` <=> NEW.`outputId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,OLD.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`outputId` <=> NEW.`outputId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 1,NEW.userId,NEW.`outputId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1488, "snapshot_journal_physical_1_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_1_DELETE", + "table": "outputs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`outputId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(1,OLD.`outputId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1737, "snapshot_journal_physical_2_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_2_INSERT", + "table": "certificates", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 2,NEW.userId,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3564, "snapshot_journal_physical_2_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_2_UPDATE", + "table": "certificates", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`certificateId` AS BINARY) <=> CAST(NEW.`certificateId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`serialNumber` AS BINARY) <=> CAST(NEW.`serialNumber` AS BINARY)) OR NOT (CAST(OLD.`type` AS BINARY) <=> CAST(NEW.`type` AS BINARY)) OR NOT (CAST(OLD.`certifier` AS BINARY) <=> CAST(NEW.`certifier` AS BINARY)) OR NOT (CAST(OLD.`subject` AS BINARY) <=> CAST(NEW.`subject` AS BINARY)) OR NOT (CAST(OLD.`verifier` AS BINARY) <=> CAST(NEW.`verifier` AS BINARY)) OR NOT (CAST(OLD.`revocationOutpoint` AS BINARY) <=> CAST(NEW.`revocationOutpoint` AS BINARY)) OR NOT (CAST(OLD.`signature` AS BINARY) <=> CAST(NEW.`signature` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`certificateId` <=> NEW.`certificateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,OLD.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`certificateId` <=> NEW.`certificateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 2,NEW.userId,NEW.`certificateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1498, "snapshot_journal_physical_2_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_2_DELETE", + "table": "certificates", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`certificateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(2,OLD.`certificateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1725, "snapshot_journal_physical_3_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_3_INSERT", + "table": "tx_labels", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 3,NEW.userId,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3042, "snapshot_journal_physical_3_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_3_UPDATE", + "table": "tx_labels", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`txLabelId` AS BINARY) <=> CAST(NEW.`txLabelId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`label` AS BINARY) <=> CAST(NEW.`label` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`txLabelId` <=> NEW.`txLabelId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,OLD.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`txLabelId` <=> NEW.`txLabelId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 3,NEW.userId,NEW.`txLabelId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1490, "snapshot_journal_physical_3_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_3_DELETE", + "table": "tx_labels", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`txLabelId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(3,OLD.`txLabelId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1722, "snapshot_journal_physical_4_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_4_INSERT", + "table": "output_baskets", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 4,NEW.userId,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3233, "snapshot_journal_physical_4_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_4_UPDATE", + "table": "output_baskets", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`basketId` AS BINARY) <=> CAST(NEW.`basketId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`name` AS BINARY) <=> CAST(NEW.`name` AS BINARY)) OR NOT (CAST(OLD.`numberOfDesiredUTXOs` AS BINARY) <=> CAST(NEW.`numberOfDesiredUTXOs` AS BINARY)) OR NOT (CAST(OLD.`minimumDesiredUTXOValue` AS BINARY) <=> CAST(NEW.`minimumDesiredUTXOValue` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`basketId` <=> NEW.`basketId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,OLD.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`basketId` <=> NEW.`basketId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 4,NEW.userId,NEW.`basketId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1488, "snapshot_journal_physical_4_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_4_DELETE", + "table": "output_baskets", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`basketId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(4,OLD.`basketId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1731, "snapshot_journal_physical_5_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_5_INSERT", + "table": "output_tags", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 5,NEW.userId,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3060, "snapshot_journal_physical_5_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_5_UPDATE", + "table": "output_tags", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`outputTagId` AS BINARY) <=> CAST(NEW.`outputTagId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`tag` AS BINARY) <=> CAST(NEW.`tag` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`outputTagId` <=> NEW.`outputTagId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,OLD.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`outputTagId` <=> NEW.`outputTagId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 5,NEW.userId,NEW.`outputTagId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1494, "snapshot_journal_physical_5_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_5_DELETE", + "table": "output_tags", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`outputTagId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(5,OLD.`outputTagId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1734, "snapshot_journal_physical_6_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_6_INSERT", + "table": "commissions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 6,NEW.userId,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3330, "snapshot_journal_physical_6_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_6_UPDATE", + "table": "commissions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`commissionId` AS BINARY) <=> CAST(NEW.`commissionId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`keyOffset` AS BINARY) <=> CAST(NEW.`keyOffset` AS BINARY)) OR NOT (CAST(OLD.`isRedeemed` AS BINARY) <=> CAST(NEW.`isRedeemed` AS BINARY)) OR NOT (CAST(OLD.`lockingScript` AS BINARY) <=> CAST(NEW.`lockingScript` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`commissionId` <=> NEW.`commissionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,OLD.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`commissionId` <=> NEW.`commissionId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 6,NEW.userId,NEW.`commissionId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1496, "snapshot_journal_physical_6_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_6_DELETE", + "table": "commissions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`commissionId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(6,OLD.`commissionId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1731, "snapshot_journal_physical_7_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_7_INSERT", + "table": "sync_states", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 7,NEW.userId,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3676, "snapshot_journal_physical_7_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_7_UPDATE", + "table": "sync_states", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`syncStateId` AS BINARY) <=> CAST(NEW.`syncStateId` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`storageIdentityKey` AS BINARY) <=> CAST(NEW.`storageIdentityKey` AS BINARY)) OR NOT (CAST(OLD.`storageName` AS BINARY) <=> CAST(NEW.`storageName` AS BINARY)) OR NOT (CAST(OLD.`status` AS BINARY) <=> CAST(NEW.`status` AS BINARY)) OR NOT (CAST(OLD.`init` AS BINARY) <=> CAST(NEW.`init` AS BINARY)) OR NOT (CAST(OLD.`refNum` AS BINARY) <=> CAST(NEW.`refNum` AS BINARY)) OR NOT (CAST(OLD.`syncMap` AS BINARY) <=> CAST(NEW.`syncMap` AS BINARY)) OR NOT (CAST(OLD.`when` AS BINARY) <=> CAST(NEW.`when` AS BINARY)) OR NOT (CAST(OLD.`satoshis` AS BINARY) <=> CAST(NEW.`satoshis` AS BINARY)) OR NOT (CAST(OLD.`errorLocal` AS BINARY) <=> CAST(NEW.`errorLocal` AS BINARY)) OR NOT (CAST(OLD.`errorOther` AS BINARY) <=> CAST(NEW.`errorOther` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (NEW.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`syncStateId` <=> NEW.`syncStateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,OLD.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`syncStateId` <=> NEW.`syncStateId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 7,NEW.userId,NEW.`syncStateId`,0,CAST('' AS BINARY),journalRevision,1 ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1494, "snapshot_journal_physical_7_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_7_DELETE", + "table": "sync_states", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`syncStateId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400) AND (OLD.userId BETWEEN 1 AND 9007199254740991)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(7,OLD.`syncStateId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1443, "snapshot_journal_physical_8_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_8_INSERT", + "table": "proven_txs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,NEW.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3014, "snapshot_journal_physical_8_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_8_UPDATE", + "table": "proven_txs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`provenTxId` AS BINARY) <=> CAST(NEW.`provenTxId` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`height` AS BINARY) <=> CAST(NEW.`height` AS BINARY)) OR NOT (CAST(OLD.`index` AS BINARY) <=> CAST(NEW.`index` AS BINARY)) OR NOT (CAST(OLD.`merklePath` AS BINARY) <=> CAST(NEW.`merklePath` AS BINARY)) OR NOT (CAST(OLD.`rawTx` AS BINARY) <=> CAST(NEW.`rawTx` AS BINARY)) OR NOT (CAST(OLD.`blockHash` AS BINARY) <=> CAST(NEW.`blockHash` AS BINARY)) OR NOT (CAST(OLD.`merkleRoot` AS BINARY) <=> CAST(NEW.`merkleRoot` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`provenTxId` <=> NEW.`provenTxId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,OLD.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,NEW.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`provenTxId` <=> NEW.`provenTxId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1444, "snapshot_journal_physical_8_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_8_DELETE", + "table": "proven_txs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`provenTxId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(8,OLD.`provenTxId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1449, "snapshot_journal_physical_9_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_9_INSERT", + "table": "proven_tx_reqs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,NEW.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3439, "snapshot_journal_physical_9_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_9_UPDATE", + "table": "proven_tx_reqs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`provenTxReqId` AS BINARY) <=> CAST(NEW.`provenTxReqId` AS BINARY)) OR NOT (CAST(OLD.`provenTxId` AS BINARY) <=> CAST(NEW.`provenTxId` AS BINARY)) OR NOT (CAST(OLD.`status` AS BINARY) <=> CAST(NEW.`status` AS BINARY)) OR NOT (CAST(OLD.`attempts` AS BINARY) <=> CAST(NEW.`attempts` AS BINARY)) OR NOT (CAST(OLD.`notified` AS BINARY) <=> CAST(NEW.`notified` AS BINARY)) OR NOT (CAST(OLD.`txid` AS BINARY) <=> CAST(NEW.`txid` AS BINARY)) OR NOT (CAST(OLD.`batch` AS BINARY) <=> CAST(NEW.`batch` AS BINARY)) OR NOT (CAST(OLD.`history` AS BINARY) <=> CAST(NEW.`history` AS BINARY)) OR NOT (CAST(OLD.`notify` AS BINARY) <=> CAST(NEW.`notify` AS BINARY)) OR NOT (CAST(OLD.`rawTx` AS BINARY) <=> CAST(NEW.`rawTx` AS BINARY)) OR NOT (CAST(OLD.`inputBEEF` AS BINARY) <=> CAST(NEW.`inputBEEF` AS BINARY)) OR NOT (CAST(OLD.`wasBroadcast` AS BINARY) <=> CAST(NEW.`wasBroadcast` AS BINARY)) OR NOT (CAST(OLD.`rebroadcastAttempts` AS BINARY) <=> CAST(NEW.`rebroadcastAttempts` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.`provenTxReqId` <=> NEW.`provenTxReqId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,OLD.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,NEW.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.`provenTxReqId` <=> NEW.`provenTxReqId`) AND (0 <=> 0) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1450, "snapshot_journal_physical_9_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_9_DELETE", + "table": "proven_tx_reqs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.`provenTxReqId` BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(9,OLD.`provenTxReqId`,0,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1883, "snapshot_journal_physical_10_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_10_INSERT", + "table": "tx_labels_map", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.txLabelId BETWEEN 1 AND 9007199254740991) AND (NEW.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,NEW.txLabelId,NEW.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 10,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=0 AND snapshotLeftId=NEW.txLabelId AND snapshotRightId=NEW.transactionId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3182, "snapshot_journal_physical_10_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_10_UPDATE", + "table": "tx_labels_map", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`txLabelId` AS BINARY) <=> CAST(NEW.`txLabelId` AS BINARY)) OR NOT (CAST(OLD.`transactionId` AS BINARY) <=> CAST(NEW.`transactionId` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.txLabelId BETWEEN 1 AND 9007199254740991) AND (NEW.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.txLabelId BETWEEN 1 AND 9007199254740991) AND (OLD.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.txLabelId <=> NEW.txLabelId) AND (OLD.transactionId <=> NEW.transactionId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,OLD.txLabelId,OLD.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,NEW.txLabelId,NEW.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.txLabelId <=> NEW.txLabelId) AND (OLD.transactionId <=> NEW.transactionId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 10,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=0 AND snapshotLeftId=NEW.txLabelId AND snapshotRightId=NEW.transactionId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1471, "snapshot_journal_physical_10_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_10_DELETE", + "table": "tx_labels_map", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.txLabelId BETWEEN 1 AND 9007199254740991) AND (OLD.transactionId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(10,OLD.txLabelId,OLD.transactionId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1874, "snapshot_journal_physical_11_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_11_INSERT", + "table": "output_tags_map", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.outputTagId BETWEEN 1 AND 9007199254740991) AND (NEW.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,NEW.outputTagId,NEW.outputId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 11,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=1 AND snapshotLeftId=NEW.outputTagId AND snapshotRightId=NEW.outputId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3149, "snapshot_journal_physical_11_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_11_UPDATE", + "table": "output_tags_map", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`outputTagId` AS BINARY) <=> CAST(NEW.`outputTagId` AS BINARY)) OR NOT (CAST(OLD.`outputId` AS BINARY) <=> CAST(NEW.`outputId` AS BINARY)) OR NOT (CAST(OLD.`isDeleted` AS BINARY) <=> CAST(NEW.`isDeleted` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.outputTagId BETWEEN 1 AND 9007199254740991) AND (NEW.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.outputTagId BETWEEN 1 AND 9007199254740991) AND (OLD.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.outputTagId <=> NEW.outputTagId) AND (OLD.outputId <=> NEW.outputId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,OLD.outputTagId,OLD.outputId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,NEW.outputTagId,NEW.outputId,CAST('' AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.outputTagId <=> NEW.outputTagId) AND (OLD.outputId <=> NEW.outputId) AND (CAST('' AS BINARY) <=> CAST('' AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 11,snapshotUserId,snapshotLeftId,snapshotRightId,CAST('' AS BINARY),journalRevision,1 FROM snapshot_relation_keys WHERE snapshotTableId=1 AND snapshotLeftId=NEW.outputTagId AND snapshotRightId=NEW.outputId AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1465, "snapshot_journal_physical_11_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_11_DELETE", + "table": "output_tags_map", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.outputTagId BETWEEN 1 AND 9007199254740991) AND (OLD.outputId BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST('' AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(11,OLD.outputTagId,OLD.outputId,CAST('' AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1885, "snapshot_journal_physical_12_INSERT"], + "rows": [ + { + "name": "snapshot_journal_physical_12_INSERT", + "table": "certificate_fields", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,NEW.certificateId,0,CAST(NEW.fieldName AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN TRUE THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,snapshotUserId,snapshotCertificateId,0,CAST(snapshotFieldName AS BINARY),journalRevision,1 FROM snapshot_certificate_field_keys WHERE snapshotCertificateId=NEW.certificateId AND snapshotFieldName=NEW.fieldName AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [3328, "snapshot_journal_physical_12_UPDATE"], + "rows": [ + { + "name": "snapshot_journal_physical_12_UPDATE", + "table": "certificate_fields", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF NOT (CAST(OLD.`created_at` AS BINARY) <=> CAST(NEW.`created_at` AS BINARY)) OR NOT (CAST(OLD.`updated_at` AS BINARY) <=> CAST(NEW.`updated_at` AS BINARY)) OR NOT (CAST(OLD.`userId` AS BINARY) <=> CAST(NEW.`userId` AS BINARY)) OR NOT (CAST(OLD.`certificateId` AS BINARY) <=> CAST(NEW.`certificateId` AS BINARY)) OR NOT (CAST(OLD.`fieldName` AS BINARY) <=> CAST(NEW.`fieldName` AS BINARY)) OR NOT (CAST(OLD.`fieldValue` AS BINARY) <=> CAST(NEW.`fieldValue` AS BINARY)) OR NOT (CAST(OLD.`masterKey` AS BINARY) <=> CAST(NEW.`masterKey` AS BINARY)) THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((NEW.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(NEW.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF NOT COALESCE(((OLD.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(OLD.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN IF NOT ((OLD.certificateId <=> NEW.certificateId) AND (0 <=> 0) AND (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY))) THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,OLD.certificateId,0,CAST(OLD.fieldName AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,NEW.certificateId,0,CAST(NEW.fieldName AS BINARY),journalRevision,journalRevision,1) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN NOT ((OLD.certificateId <=> NEW.certificateId) AND (0 <=> 0) AND (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY))) THEN VALUES(generation) ELSE generation END,present=VALUES(present); INSERT INTO snapshot_journal_scope(tableId,userId,id1,id2,exactText,revision,present) SELECT 12,snapshotUserId,snapshotCertificateId,0,CAST(snapshotFieldName AS BINARY),journalRevision,1 FROM snapshot_certificate_field_keys WHERE snapshotCertificateId=NEW.certificateId AND snapshotFieldName=NEW.fieldName AND snapshotMembership<>0 FOR SHARE ON DUPLICATE KEY UPDATE revision=VALUES(revision),present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT TRIGGER_NAME name,EVENT_OBJECT_TABLE `table`,EVENT_MANIPULATION event,ACTION_TIMING timing,SUBSTRING(ACTION_STATEMENT,1,?) body,SQL_MODE sqlMode,CHARACTER_SET_CLIENT charset,COLLATION_CONNECTION collation,DATABASE_COLLATION databaseCollation,DEFINER definer FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND TRIGGER_NAME=?", + "bindings": [1469, "snapshot_journal_physical_12_DELETE"], + "rows": [ + { + "name": "snapshot_journal_physical_12_DELETE", + "table": "certificate_fields", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN /* snapshot-journal-owner:338b08b7-6fd5-4369-a892-06aa53896066 */ DECLARE journalCeiling BIGINT UNSIGNED;\nDECLARE journalRevision BIGINT UNSIGNED;\nDECLARE journalEnabled BOOLEAN; IF TRUE THEN SELECT ceiling INTO journalCeiling FROM snapshot_journal_clock WHERE id=1 FOR SHARE;\nSELECT NOT EXISTS(SELECT 1 FROM snapshot_journal_invalid WHERE id=1) INTO journalEnabled;\nIF journalEnabled THEN\n INSERT INTO snapshot_journal_events() VALUES();\n SET journalRevision=LAST_INSERT_ID();\n DELETE FROM snapshot_journal_events WHERE revision=journalRevision;\n IF journalRevision>journalCeiling THEN\n INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,IF(journalRevision>9223372036854775807,'revision-exhausted','capacity-exhausted'));\n SET journalEnabled=FALSE;\n END IF;\nEND IF; IF NOT COALESCE(((OLD.certificateId BETWEEN 1 AND 9007199254740991) AND (0 BETWEEN 0 AND 9007199254740991) AND (OCTET_LENGTH(CAST(OLD.fieldName AS BINARY))<=400)),FALSE) THEN INSERT IGNORE INTO snapshot_journal_invalid(id,reason) VALUES(1,'key-out-of-range'); SET journalEnabled=FALSE; END IF; IF journalEnabled THEN INSERT INTO snapshot_journal_physical(tableId,id1,id2,exactText,revision,generation,present) VALUES(12,OLD.certificateId,0,CAST(OLD.fieldName AS BINARY),journalRevision,journalRevision,0) ON DUPLICATE KEY UPDATE revision=VALUES(revision),generation=CASE WHEN FALSE THEN VALUES(generation) ELSE generation END,present=VALUES(present); END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "collation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci", + "definer": "root@%" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["transactions"], + "rows": [ + { + "name": "idx_transactions_nosend_expiry", + "field": "noSendExpiryState" + }, + { + "name": "idx_transactions_nosend_expiry", + "field": "noSendExpiryDeadline" + }, + { + "name": "idx_transactions_nosend_reclaim", + "field": "userId" + }, + { + "name": "idx_transactions_nosend_reclaim", + "field": "noSendExpiryReclaimTxid" + }, + { + "name": "idx_transactions_user_proven_tx", + "field": "userId" + }, + { + "name": "idx_transactions_user_proven_tx", + "field": "provenTxId" + }, + { + "name": "idx_transactions_user_txid", + "field": "userId" + }, + { + "name": "idx_transactions_user_txid", + "field": "txid" + }, + { + "name": "PRIMARY", + "field": "transactionId" + }, + { + "name": "transactions_proventxid_foreign", + "field": "provenTxId" + }, + { + "name": "transactions_reference_unique", + "field": "reference" + }, + { + "name": "transactions_status_index", + "field": "status" + }, + { + "name": "transactions_txid_index", + "field": "txid" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["outputs"], + "rows": [ + { + "name": "idx_outputs_funding_selection", + "field": "userId" + }, + { + "name": "idx_outputs_funding_selection", + "field": "basketId" + }, + { + "name": "idx_outputs_funding_selection", + "field": "spendable" + }, + { + "name": "idx_outputs_funding_selection", + "field": "spentBy" + }, + { + "name": "idx_outputs_funding_selection", + "field": "satoshis" + }, + { + "name": "idx_outputs_funding_selection", + "field": "outputId" + }, + { + "name": "idx_outputs_spentby", + "field": "spentBy" + }, + { + "name": "idx_outputs_user_basket_spendable_outputid", + "field": "userId" + }, + { + "name": "idx_outputs_user_basket_spendable_outputid", + "field": "basketId" + }, + { + "name": "idx_outputs_user_basket_spendable_outputid", + "field": "spendable" + }, + { + "name": "idx_outputs_user_basket_spendable_outputid", + "field": "outputId" + }, + { + "name": "idx_outputs_user_basket_spendable_satoshis", + "field": "userId" + }, + { + "name": "idx_outputs_user_basket_spendable_satoshis", + "field": "basketId" + }, + { + "name": "idx_outputs_user_basket_spendable_satoshis", + "field": "spendable" + }, + { + "name": "idx_outputs_user_basket_spendable_satoshis", + "field": "satoshis" + }, + { + "name": "idx_outputs_user_spendable_outputid", + "field": "userId" + }, + { + "name": "idx_outputs_user_spendable_outputid", + "field": "spendable" + }, + { + "name": "idx_outputs_user_spendable_outputid", + "field": "outputId" + }, + { + "name": "outputs_basketid_foreign", + "field": "basketId" + }, + { + "name": "outputs_spendable_index", + "field": "spendable" + }, + { + "name": "outputs_transactionid_vout_userid_unique", + "field": "transactionId" + }, + { + "name": "outputs_transactionid_vout_userid_unique", + "field": "vout" + }, + { + "name": "outputs_transactionid_vout_userid_unique", + "field": "userId" + }, + { + "name": "PRIMARY", + "field": "outputId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["certificates"], + "rows": [ + { + "name": "certificates_userid_type_certifier_serialnumber_unique", + "field": "userId" + }, + { + "name": "certificates_userid_type_certifier_serialnumber_unique", + "field": "type" + }, + { + "name": "certificates_userid_type_certifier_serialnumber_unique", + "field": "certifier" + }, + { + "name": "certificates_userid_type_certifier_serialnumber_unique", + "field": "serialNumber" + }, + { + "name": "PRIMARY", + "field": "certificateId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["tx_labels"], + "rows": [ + { + "name": "PRIMARY", + "field": "txLabelId" + }, + { + "name": "tx_labels_label_userid_unique", + "field": "label" + }, + { + "name": "tx_labels_label_userid_unique", + "field": "userId" + }, + { + "name": "tx_labels_userid_foreign", + "field": "userId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["output_baskets"], + "rows": [ + { + "name": "output_baskets_name_userid_unique", + "field": "name" + }, + { + "name": "output_baskets_name_userid_unique", + "field": "userId" + }, + { + "name": "output_baskets_userid_foreign", + "field": "userId" + }, + { + "name": "PRIMARY", + "field": "basketId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["output_tags"], + "rows": [ + { + "name": "output_tags_tag_userid_unique", + "field": "tag" + }, + { + "name": "output_tags_tag_userid_unique", + "field": "userId" + }, + { + "name": "output_tags_userid_foreign", + "field": "userId" + }, + { + "name": "PRIMARY", + "field": "outputTagId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["commissions"], + "rows": [ + { + "name": "commissions_transactionid_index", + "field": "transactionId" + }, + { + "name": "commissions_transactionid_unique", + "field": "transactionId" + }, + { + "name": "commissions_userid_foreign", + "field": "userId" + }, + { + "name": "PRIMARY", + "field": "commissionId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["sync_states"], + "rows": [ + { + "name": "PRIMARY", + "field": "syncStateId" + }, + { + "name": "sync_states_refnum_index", + "field": "refNum" + }, + { + "name": "sync_states_refnum_unique", + "field": "refNum" + }, + { + "name": "sync_states_status_index", + "field": "status" + }, + { + "name": "sync_states_userid_foreign", + "field": "userId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["proven_txs"], + "rows": [ + { + "name": "PRIMARY", + "field": "provenTxId" + }, + { + "name": "proven_txs_blockhash_index", + "field": "blockHash" + }, + { + "name": "proven_txs_txid_unique", + "field": "txid" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["proven_tx_reqs"], + "rows": [ + { + "name": "PRIMARY", + "field": "provenTxReqId" + }, + { + "name": "proven_tx_reqs_batch_index", + "field": "batch" + }, + { + "name": "proven_tx_reqs_proventxid_foreign", + "field": "provenTxId" + }, + { + "name": "proven_tx_reqs_status_index", + "field": "status" + }, + { + "name": "proven_tx_reqs_txid_index", + "field": "txid" + }, + { + "name": "proven_tx_reqs_txid_unique", + "field": "txid" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["tx_labels_map"], + "rows": [ + { + "name": "idx_tx_labels_map_tx_deleted", + "field": "transactionId" + }, + { + "name": "idx_tx_labels_map_tx_deleted", + "field": "isDeleted" + }, + { + "name": "tx_labels_map_transactionid_index", + "field": "transactionId" + }, + { + "name": "tx_labels_map_txlabelid_transactionid_unique", + "field": "txLabelId" + }, + { + "name": "tx_labels_map_txlabelid_transactionid_unique", + "field": "transactionId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["output_tags_map"], + "rows": [ + { + "name": "idx_output_tags_map_output_deleted_tag", + "field": "outputId" + }, + { + "name": "idx_output_tags_map_output_deleted_tag", + "field": "isDeleted" + }, + { + "name": "idx_output_tags_map_output_deleted_tag", + "field": "outputTagId" + }, + { + "name": "output_tags_map_outputid_index", + "field": "outputId" + }, + { + "name": "output_tags_map_outputtagid_outputid_unique", + "field": "outputTagId" + }, + { + "name": "output_tags_map_outputtagid_outputid_unique", + "field": "outputId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["certificate_fields"], + "rows": [ + { + "name": "certificate_fields_certificateid_foreign", + "field": "certificateId" + }, + { + "name": "certificate_fields_fieldname_certificateid_unique", + "field": "fieldName" + }, + { + "name": "certificate_fields_fieldname_certificateid_unique", + "field": "certificateId" + }, + { + "name": "certificate_fields_userid_foreign", + "field": "userId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_profile_keys"], + "rows": [ + { + "name": "PRIMARY", + "field": "snapshotTableId" + }, + { + "name": "PRIMARY", + "field": "snapshotUserId" + }, + { + "name": "PRIMARY", + "field": "snapshotRowId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_relation_keys"], + "rows": [ + { + "name": "PRIMARY", + "field": "snapshotTableId" + }, + { + "name": "PRIMARY", + "field": "snapshotUserId" + }, + { + "name": "PRIMARY", + "field": "snapshotLeftId" + }, + { + "name": "PRIMARY", + "field": "snapshotRightId" + }, + { + "name": "snapshot_relation_map", + "field": "snapshotTableId" + }, + { + "name": "snapshot_relation_map", + "field": "snapshotLeftId" + }, + { + "name": "snapshot_relation_map", + "field": "snapshotRightId" + }, + { + "name": "snapshot_relation_map", + "field": "snapshotUserId" + }, + { + "name": "snapshot_relation_right", + "field": "snapshotTableId" + }, + { + "name": "snapshot_relation_right", + "field": "snapshotUserId" + }, + { + "name": "snapshot_relation_right", + "field": "snapshotRightId" + }, + { + "name": "snapshot_relation_right", + "field": "snapshotLeftId" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_certificate_field_keys"], + "rows": [ + { + "name": "PRIMARY", + "field": "snapshotUserId" + }, + { + "name": "PRIMARY", + "field": "snapshotFieldName" + }, + { + "name": "PRIMARY", + "field": "snapshotCertificateId" + }, + { + "name": "snapshot_certificate_lookup", + "field": "snapshotFieldName" + }, + { + "name": "snapshot_certificate_lookup", + "field": "snapshotCertificateId" + }, + { + "name": "snapshot_certificate_lookup", + "field": "snapshotUserId" + }, + { + "name": "snapshot_certificate_parent", + "field": "snapshotCertificateId" + }, + { + "name": "snapshot_certificate_parent", + "field": "snapshotUserId" + }, + { + "name": "snapshot_certificate_parent", + "field": "snapshotFieldName" + } + ] + }, + { + "sql": "SELECT INDEX_NAME name,COLUMN_NAME field FROM information_schema.STATISTICS WHERE TABLE_SCHEMA=DATABASE() AND TABLE_NAME=? ORDER BY INDEX_NAME,SEQ_IN_INDEX", + "bindings": ["snapshot_global_keys"], + "rows": [ + { + "name": "PRIMARY", + "field": "tableId" + }, + { + "name": "PRIMARY", + "field": "userId" + }, + { + "name": "PRIMARY", + "field": "rowId" + }, + { + "name": "snapshot_global_page", + "field": "tableId" + }, + { + "name": "snapshot_global_page", + "field": "userId" + }, + { + "name": "snapshot_global_page", + "field": "present" + }, + { + "name": "snapshot_global_page", + "field": "rowId" + }, + { + "name": "snapshot_global_target", + "field": "tableId" + }, + { + "name": "snapshot_global_target", + "field": "rowId" + }, + { + "name": "snapshot_global_target", + "field": "userId" + } + ] + } +] diff --git a/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-state-fixture.json b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-state-fixture.json new file mode 100644 index 000000000..5fd168ef9 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-generation-state-fixture.json @@ -0,0 +1,10 @@ +{ + "id": 1, + "version": 1, + "epoch": "338b08b7-6fd5-4369-a892-06aa53896066", + "source": "166b30f8f3e2c27b0bdc36a22b1aa86514cd245a3ad0ce84f69c215890a3b2ee", + "plan": "8d32f91f5cf64eb06b842835b18ebd86121a90388efa3e219cbddb71c0320309", + "ceiling": "9223372036854775807", + "nextObject": 59, + "complete": 1 +} diff --git a/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-intent-metadata-fixture.json b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-intent-metadata-fixture.json new file mode 100644 index 000000000..62159c24a --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-intent-metadata-fixture.json @@ -0,0 +1,112 @@ +{ + "tables": [ + { + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "ascii_bin", + "rowFormat": "Dynamic", + "options": "row_format=DYNAMIC" + } + ], + "columns": [ + { + "name": "id", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "version", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "epoch", + "type": "varchar(36)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "source", + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "plan", + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "ceiling", + "type": "varchar(19)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "ascii", + "collation": "ascii_bin", + "expression": "" + }, + { + "name": "nextObject", + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "name": "complete", + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + } + ], + "indexes": [ + { + "name": "PRIMARY", + "columnName": "id", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "type": "BTREE", + "visible": "YES" + } + ], + "constraints": [ + { + "name": "PRIMARY", + "type": "PRIMARY KEY", + "enforced": "YES" + } + ], + "triggers": [] +} diff --git a/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-source-metadata-fixture.json b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-source-metadata-fixture.json new file mode 100644 index 000000000..2cacf9b94 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/fixtures/snapshotJournal/mysql-source-metadata-fixture.json @@ -0,0 +1,5989 @@ +{ + "sourceTables": [ + { + "name": "certificate_fields", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "certificates", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "commissions", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "output_baskets", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "output_tags", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "output_tags_map", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "outputs", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "proven_tx_reqs", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "proven_txs", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "sync_states", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "transactions", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "tx_labels", + "engine": "InnoDB", + "type": "BASE TABLE" + }, + { + "name": "tx_labels_map", + "engine": "InnoDB", + "type": "BASE TABLE" + } + ], + "sourceRules": [ + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + } + ], + "sourceTriggers": [ + { + "name": "snapshot_profile_0_delete", + "table": "transactions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; END" + }, + { + "name": "snapshot_profile_0_update", + "table": "transactions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END IF; END" + }, + { + "name": "snapshot_profile_0_insert", + "table": "transactions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END" + }, + { + "name": "snapshot_profile_1_delete", + "table": "outputs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; END" + }, + { + "name": "snapshot_profile_1_update", + "table": "outputs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END IF; END" + }, + { + "name": "snapshot_profile_1_insert", + "table": "outputs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END" + }, + { + "name": "snapshot_profile_2_delete", + "table": "certificates", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; END" + }, + { + "name": "snapshot_profile_2_update", + "table": "certificates", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END IF; END" + }, + { + "name": "snapshot_profile_2_insert", + "table": "certificates", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END" + }, + { + "name": "snapshot_profile_3_delete", + "table": "tx_labels", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; END" + }, + { + "name": "snapshot_profile_3_update", + "table": "tx_labels", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txLabelId <=> NEW.txLabelId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END IF; END" + }, + { + "name": "snapshot_profile_3_insert", + "table": "tx_labels", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END" + }, + { + "name": "snapshot_profile_4_delete", + "table": "output_baskets", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; END" + }, + { + "name": "snapshot_profile_4_update", + "table": "output_baskets", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.basketId <=> NEW.basketId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END IF; END" + }, + { + "name": "snapshot_profile_4_insert", + "table": "output_baskets", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END" + }, + { + "name": "snapshot_profile_5_delete", + "table": "output_tags", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; END" + }, + { + "name": "snapshot_profile_5_update", + "table": "output_tags", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputTagId <=> NEW.outputTagId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END IF; END" + }, + { + "name": "snapshot_profile_5_insert", + "table": "output_tags", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END" + }, + { + "name": "snapshot_profile_6_delete", + "table": "commissions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; END" + }, + { + "name": "snapshot_profile_6_update", + "table": "commissions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.commissionId <=> NEW.commissionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END IF; END" + }, + { + "name": "snapshot_profile_6_insert", + "table": "commissions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END" + }, + { + "name": "snapshot_profile_7_delete", + "table": "sync_states", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; END" + }, + { + "name": "snapshot_profile_7_update", + "table": "sync_states", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.syncStateId <=> NEW.syncStateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END IF; END" + }, + { + "name": "snapshot_profile_7_insert", + "table": "sync_states", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END" + }, + { + "name": "snapshot_relation_0_map_delete", + "table": "tx_labels_map", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END" + }, + { + "name": "snapshot_relation_0_map_before_update", + "table": "tx_labels_map", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END IF; END" + }, + { + "name": "snapshot_relation_0_left_delete", + "table": "tx_labels", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_relation_0_left_before_update", + "table": "tx_labels", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_relation_0_right_delete", + "table": "transactions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_relation_0_right_before_update", + "table": "transactions", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_relation_1_map_delete", + "table": "output_tags_map", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END" + }, + { + "name": "snapshot_relation_1_map_before_update", + "table": "output_tags_map", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END IF; END" + }, + { + "name": "snapshot_relation_1_left_delete", + "table": "output_tags", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_relation_1_left_before_update", + "table": "output_tags", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_relation_1_right_delete", + "table": "outputs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_relation_1_right_before_update", + "table": "outputs", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_relation_0_map_insert", + "table": "tx_labels_map", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_relation_0_map_after_update", + "table": "tx_labels_map", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_relation_0_left_insert", + "table": "tx_labels", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END" + }, + { + "name": "snapshot_relation_0_left_after_update", + "table": "tx_labels", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END" + }, + { + "name": "snapshot_relation_0_right_insert", + "table": "transactions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_relation_0_right_after_update", + "table": "transactions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_relation_1_map_insert", + "table": "output_tags_map", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_relation_1_map_after_update", + "table": "output_tags_map", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_relation_1_left_insert", + "table": "output_tags", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END" + }, + { + "name": "snapshot_relation_1_left_after_update", + "table": "output_tags", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END" + }, + { + "name": "snapshot_relation_1_right_insert", + "table": "outputs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END" + }, + { + "name": "snapshot_relation_1_right_after_update", + "table": "outputs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_certificate_field_delete", + "table": "certificate_fields", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END" + }, + { + "name": "snapshot_certificate_field_before_update", + "table": "certificate_fields", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END IF; END" + }, + { + "name": "snapshot_certificate_parent_delete", + "table": "certificates", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END" + }, + { + "name": "snapshot_certificate_parent_before_update", + "table": "certificates", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END IF; END" + }, + { + "name": "snapshot_certificate_field_insert", + "table": "certificate_fields", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END" + }, + { + "name": "snapshot_certificate_field_after_update", + "table": "certificate_fields", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_certificate_parent_insert", + "table": "certificates", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END" + }, + { + "name": "snapshot_certificate_parent_after_update", + "table": "certificates", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END" + }, + { + "name": "snapshot_global_edge_delete", + "table": "snapshot_global_edges", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN UPDATE snapshot_global_keys SET refs=refs-1 WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId; DELETE FROM snapshot_global_keys WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId AND refs=0; DELETE FROM snapshot_global_guards WHERE proofId=OLD.rowId AND OLD.tableId=1 AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.rowId); END" + }, + { + "name": "snapshot_global_edge_insert", + "table": "snapshot_global_edges", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_global_keys (tableId,userId,rowId,refs,present) VALUES (NEW.tableId,NEW.userId,NEW.rowId,1,CASE WHEN NEW.tableId=0 THEN 1 ELSE (SELECT present FROM snapshot_global_guards WHERE proofId=NEW.rowId FOR SHARE) END) ON DUPLICATE KEY UPDATE refs=snapshot_global_keys.refs+1; END" + }, + { + "name": "snapshot_global_tx_delete", + "table": "transactions", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END" + }, + { + "name": "snapshot_global_tx_before_update", + "table": "transactions", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END IF; END" + }, + { + "name": "snapshot_global_req_delete", + "table": "proven_tx_reqs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END" + }, + { + "name": "snapshot_global_req_before_update", + "table": "proven_tx_reqs", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END IF; END" + }, + { + "name": "snapshot_global_proof_delete", + "table": "proven_txs", + "event": "DELETE", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END" + }, + { + "name": "snapshot_global_proof_before_update", + "table": "proven_txs", + "event": "UPDATE", + "timing": "BEFORE", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END IF; END" + }, + { + "name": "snapshot_global_proof_insert", + "table": "proven_txs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END" + }, + { + "name": "snapshot_global_proof_after_update", + "table": "proven_txs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END" + }, + { + "name": "snapshot_global_tx_insert", + "table": "transactions", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END" + }, + { + "name": "snapshot_global_tx_after_update", + "table": "transactions", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END IF; END" + }, + { + "name": "snapshot_global_req_insert", + "table": "proven_tx_reqs", + "event": "INSERT", + "timing": "AFTER", + "body": "BEGIN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END" + }, + { + "name": "snapshot_global_req_after_update", + "table": "proven_tx_reqs", + "event": "UPDATE", + "timing": "AFTER", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END" + } + ], + "tables": [ + { + "name": "certificate_fields", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "certificates", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "commissions", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "output_baskets", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "output_tags", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "output_tags_map", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "outputs", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "proven_tx_reqs", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "proven_txs", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "settings", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_certificate_field_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_global_edges", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_global_guards", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_global_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_profile_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "snapshot_relation_keys", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "sync_states", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "transactions", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "tx_labels", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "tx_labels_map", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + }, + { + "name": "users", + "engine": "InnoDB", + "type": "BASE TABLE", + "collation": "utf8mb4_0900_ai_ci", + "rowFormat": "Dynamic", + "options": "" + } + ], + "columns": [ + { + "tableName": "certificate_fields", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "userId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "certificateId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "fieldName", + "position": 5, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "fieldValue", + "position": 6, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificate_fields", + "name": "masterKey", + "position": 7, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": "", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "certificateId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "certificates", + "name": "serialNumber", + "position": 5, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "type", + "position": 6, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "certifier", + "position": 7, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "subject", + "position": 8, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "verifier", + "position": 9, + "type": "varchar(100)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "revocationOutpoint", + "position": 10, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "signature", + "position": 11, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "certificates", + "name": "isDeleted", + "position": 12, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "commissionId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "transactionId", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "satoshis", + "position": 6, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "keyOffset", + "position": 7, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "commissions", + "name": "isRedeemed", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "commissions", + "name": "lockingScript", + "position": 9, + "type": "blob", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "basketId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "name", + "position": 5, + "type": "varchar(300)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "numberOfDesiredUTXOs", + "position": 6, + "type": "int", + "nullable": "NO", + "defaultValue": "144", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "minimumDesiredUTXOValue", + "position": 7, + "type": "int", + "nullable": "NO", + "defaultValue": "5000", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_baskets", + "name": "isDeleted", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "outputTagId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags", + "name": "tag", + "position": 5, + "type": "varchar(150)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "output_tags", + "name": "isDeleted", + "position": 6, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "outputTagId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "outputId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "output_tags_map", + "name": "isDeleted", + "position": 5, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "outputId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "transactionId", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "basketId", + "position": 6, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "spendable", + "position": 7, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "change", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "vout", + "position": 9, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "satoshis", + "position": 10, + "type": "bigint", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "providedBy", + "position": 11, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "purpose", + "position": 12, + "type": "varchar(20)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "type", + "position": 13, + "type": "varchar(50)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "outputDescription", + "position": 14, + "type": "varchar(2048)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "txid", + "position": 15, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "senderIdentityKey", + "position": 16, + "type": "varchar(130)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "derivationPrefix", + "position": 17, + "type": "varchar(200)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "derivationSuffix", + "position": 18, + "type": "varchar(200)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "customInstructions", + "position": 19, + "type": "varchar(2500)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "spentBy", + "position": 20, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "sequenceNumber", + "position": 21, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "spendingDescription", + "position": 22, + "type": "varchar(2048)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "outputs", + "name": "scriptLength", + "position": 23, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "scriptOffset", + "position": 24, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "outputs", + "name": "lockingScript", + "position": 25, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "provenTxReqId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "provenTxId", + "position": 4, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "status", + "position": 5, + "type": "varchar(16)", + "nullable": "NO", + "defaultValue": "unknown", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "attempts", + "position": 6, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "notified", + "position": 7, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "txid", + "position": 8, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "batch", + "position": 9, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "history", + "position": 10, + "type": "longtext", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "notify", + "position": 11, + "type": "longtext", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "rawTx", + "position": 12, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "inputBEEF", + "position": 13, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "wasBroadcast", + "position": 14, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_tx_reqs", + "name": "rebroadcastAttempts", + "position": 15, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "provenTxId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "txid", + "position": 4, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "height", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "index", + "position": 6, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "merklePath", + "position": 7, + "type": "blob", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "rawTx", + "position": 8, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "blockHash", + "position": 9, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "proven_txs", + "name": "merkleRoot", + "position": 10, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "settings", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "settings", + "name": "storageIdentityKey", + "position": 3, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "storageName", + "position": 4, + "type": "varchar(128)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "chain", + "position": 5, + "type": "varchar(10)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "dbtype", + "position": 6, + "type": "varchar(10)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "settings", + "name": "maxOutputScript", + "position": 7, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotUserId", + "position": 1, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotFieldName", + "position": 2, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotCertificateId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshotMembership", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "transactionId", + "position": 1, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "requestId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "tableId", + "position": 3, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "rowId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_edges", + "name": "userId", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_guards", + "name": "proofId", + "position": 1, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_guards", + "name": "present", + "position": 2, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "tableId", + "position": 1, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "userId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "rowId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "refs", + "position": 4, + "type": "bigint unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_global_keys", + "name": "present", + "position": 5, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_profile_keys", + "name": "snapshotTableId", + "position": 1, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_profile_keys", + "name": "snapshotUserId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_profile_keys", + "name": "snapshotRowId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotTableId", + "position": 1, + "type": "int", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotUserId", + "position": 2, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotLeftId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotRightId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshotMembership", + "position": 5, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "syncStateId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "storageIdentityKey", + "position": 5, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": "", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "storageName", + "position": 6, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "status", + "position": 7, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": "unknown", + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "init", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "refNum", + "position": 9, + "type": "varchar(100)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "syncMap", + "position": 10, + "type": "longtext", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "when", + "position": 11, + "type": "datetime", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "satoshis", + "position": 12, + "type": "bigint", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "sync_states", + "name": "errorLocal", + "position": 13, + "type": "longtext", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "sync_states", + "name": "errorOther", + "position": 14, + "type": "longtext", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "transactionId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "provenTxId", + "position": 5, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "status", + "position": 6, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "reference", + "position": 7, + "type": "varchar(64)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "isOutgoing", + "position": 8, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "satoshis", + "position": 9, + "type": "bigint", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "version", + "position": 10, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "lockTime", + "position": 11, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "description", + "position": 12, + "type": "varchar(2048)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "txid", + "position": 13, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "inputBEEF", + "position": 14, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "rawTx", + "position": 15, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryMode", + "position": 16, + "type": "varchar(16)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryValue", + "position": 17, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryDeadline", + "position": 18, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryState", + "position": 19, + "type": "varchar(24)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryAnchorTxid", + "position": 20, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryAnchorVout", + "position": 21, + "type": "int unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReleasedAt", + "position": 22, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryObservedAt", + "position": 23, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimTxid", + "position": 24, + "type": "varchar(64)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimRawTx", + "position": 25, + "type": "longblob", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimDerivationPrefix", + "position": 26, + "type": "varchar(32)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimDerivationSuffix", + "position": 27, + "type": "varchar(32)", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "transactions", + "name": "noSendExpiryReclaimSatoshis", + "position": 28, + "type": "bigint unsigned", + "nullable": "YES", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "txLabelId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "userId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "label", + "position": 5, + "type": "varchar(300)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "tx_labels", + "name": "isDeleted", + "position": 6, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "txLabelId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "transactionId", + "position": 4, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "tx_labels_map", + "name": "isDeleted", + "position": 5, + "type": "tinyint(1)", + "nullable": "NO", + "defaultValue": "0", + "extra": "", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "created_at", + "position": 1, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "updated_at", + "position": 2, + "type": "timestamp(3)", + "nullable": "NO", + "defaultValue": "CURRENT_TIMESTAMP(3)", + "extra": "DEFAULT_GENERATED", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "userId", + "position": 3, + "type": "int unsigned", + "nullable": "NO", + "defaultValue": null, + "extra": "auto_increment", + "charset": null, + "collation": null, + "expression": "" + }, + { + "tableName": "users", + "name": "identityKey", + "position": 4, + "type": "varchar(130)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + }, + { + "tableName": "users", + "name": "activeStorage", + "position": 5, + "type": "varchar(255)", + "nullable": "NO", + "defaultValue": null, + "extra": "", + "charset": "utf8mb4", + "collation": "utf8mb4_0900_ai_ci", + "expression": "" + } + ], + "indexes": [ + { + "tableName": "certificate_fields", + "name": "certificate_fields_certificateid_foreign", + "position": 1, + "columnName": "certificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_fieldname_certificateid_unique", + "position": 1, + "columnName": "fieldName", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_fieldname_certificateid_unique", + "position": 2, + "columnName": "certificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 1, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 2, + "columnName": "type", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 3, + "columnName": "certifier", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "certificates_userid_type_certifier_serialnumber_unique", + "position": 4, + "columnName": "serialNumber", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "certificates", + "name": "PRIMARY", + "position": 1, + "columnName": "certificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_index", + "position": 1, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_unique", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "commissions_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "commissions", + "name": "PRIMARY", + "position": 1, + "columnName": "commissionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "output_baskets_name_userid_unique", + "position": 1, + "columnName": "name", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "output_baskets_name_userid_unique", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "output_baskets_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_baskets", + "name": "PRIMARY", + "position": 1, + "columnName": "basketId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "output_tags_tag_userid_unique", + "position": 1, + "columnName": "tag", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "output_tags_tag_userid_unique", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "output_tags_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags", + "name": "PRIMARY", + "position": 1, + "columnName": "outputTagId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "position": 1, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "position": 2, + "columnName": "isDeleted", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "idx_output_tags_map_output_deleted_tag", + "position": 3, + "columnName": "outputTagId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputid_index", + "position": 1, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_outputid_unique", + "position": 1, + "columnName": "outputTagId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_outputid_unique", + "position": 2, + "columnName": "outputId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 2, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 3, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 4, + "columnName": "spentBy", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 5, + "columnName": "satoshis", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_funding_selection", + "position": 6, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_spentby", + "position": 1, + "columnName": "spentBy", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 2, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 3, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_outputid", + "position": 4, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 2, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 3, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_basket_spendable_satoshis", + "position": 4, + "columnName": "satoshis", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "position": 2, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "idx_outputs_user_spendable_outputid", + "position": 3, + "columnName": "outputId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_basketid_foreign", + "position": 1, + "columnName": "basketId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_spendable_index", + "position": 1, + "columnName": "spendable", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "position": 2, + "columnName": "vout", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_vout_userid_unique", + "position": 3, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "outputs", + "name": "PRIMARY", + "position": 1, + "columnName": "outputId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "PRIMARY", + "position": 1, + "columnName": "provenTxReqId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_batch_index", + "position": 1, + "columnName": "batch", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_status_index", + "position": 1, + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_txid_index", + "position": 1, + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_txid_unique", + "position": 1, + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_txs", + "name": "PRIMARY", + "position": 1, + "columnName": "provenTxId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_txs", + "name": "proven_txs_blockhash_index", + "position": 1, + "columnName": "blockHash", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "proven_txs", + "name": "proven_txs_txid_unique", + "position": 1, + "columnName": "txid", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "snapshotFieldName", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "snapshotCertificateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_lookup", + "position": 1, + "columnName": "snapshotFieldName", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_lookup", + "position": 2, + "columnName": "snapshotCertificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_lookup", + "position": 3, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_parent", + "position": 1, + "columnName": "snapshotCertificateId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_parent", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_certificate_field_keys", + "name": "snapshot_certificate_parent", + "position": 3, + "columnName": "snapshotFieldName", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 2, + "columnName": "requestId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 3, + "columnName": "tableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "PRIMARY", + "position": 4, + "columnName": "rowId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_request", + "position": 1, + "columnName": "requestId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_request", + "position": 2, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_guards", + "name": "PRIMARY", + "position": 1, + "columnName": "proofId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "tableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "rowId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 1, + "columnName": "tableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 2, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 3, + "columnName": "present", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_page", + "position": 4, + "columnName": "rowId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_target", + "position": 1, + "columnName": "tableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_target", + "position": 2, + "columnName": "rowId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_global_keys", + "name": "snapshot_global_target", + "position": 3, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_profile_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_profile_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_profile_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "snapshotRowId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 3, + "columnName": "snapshotLeftId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "PRIMARY", + "position": 4, + "columnName": "snapshotRightId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 2, + "columnName": "snapshotLeftId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 3, + "columnName": "snapshotRightId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_map", + "position": 4, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 1, + "columnName": "snapshotTableId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 2, + "columnName": "snapshotUserId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 3, + "columnName": "snapshotRightId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "snapshot_relation_keys", + "name": "snapshot_relation_right", + "position": 4, + "columnName": "snapshotLeftId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "PRIMARY", + "position": 1, + "columnName": "syncStateId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_refnum_index", + "position": 1, + "columnName": "refNum", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_refnum_unique", + "position": 1, + "columnName": "refNum", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_status_index", + "position": 1, + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "sync_states", + "name": "sync_states_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_expiry", + "position": 1, + "columnName": "noSendExpiryState", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_expiry", + "position": 2, + "columnName": "noSendExpiryDeadline", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_reclaim", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_nosend_reclaim", + "position": 2, + "columnName": "noSendExpiryReclaimTxid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_proven_tx", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_proven_tx", + "position": 2, + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_txid", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "idx_transactions_user_txid", + "position": 2, + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "PRIMARY", + "position": 1, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_reference_unique", + "position": 1, + "columnName": "reference", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_status_index", + "position": 1, + "columnName": "status", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "transactions", + "name": "transactions_txid_index", + "position": 1, + "columnName": "txid", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "YES", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "PRIMARY", + "position": 1, + "columnName": "txLabelId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "tx_labels_label_userid_unique", + "position": 1, + "columnName": "label", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "tx_labels_label_userid_unique", + "position": 2, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels", + "name": "tx_labels_userid_foreign", + "position": 1, + "columnName": "userId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "idx_tx_labels_map_tx_deleted", + "position": 1, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "idx_tx_labels_map_tx_deleted", + "position": 2, + "columnName": "isDeleted", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_transactionid_index", + "position": 1, + "columnName": "transactionId", + "nonUnique": 1, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_transactionid_unique", + "position": 1, + "columnName": "txLabelId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_transactionid_unique", + "position": 2, + "columnName": "transactionId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "users", + "name": "PRIMARY", + "position": 1, + "columnName": "userId", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + }, + { + "tableName": "users", + "name": "users_identitykey_unique", + "position": 1, + "columnName": "identityKey", + "nonUnique": 0, + "direction": "A", + "prefix": null, + "nullable": "", + "type": "BTREE", + "visible": "YES", + "expression": null + } + ], + "foreignKeys": [ + { + "tableName": "certificate_fields", + "name": "certificate_fields_certificateid_foreign", + "position": 1, + "columnName": "certificateId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "certificates", + "foreignColumn": "certificateId" + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "certificates", + "name": "certificates_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_foreign", + "position": 1, + "columnName": "transactionId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "commissions", + "name": "commissions_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "output_baskets", + "name": "output_baskets_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "output_tags", + "name": "output_tags_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputid_foreign", + "position": 1, + "columnName": "outputId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "outputs", + "foreignColumn": "outputId" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_foreign", + "position": 1, + "columnName": "outputTagId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "output_tags", + "foreignColumn": "outputTagId" + }, + { + "tableName": "outputs", + "name": "outputs_basketid_foreign", + "position": 1, + "columnName": "basketId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "output_baskets", + "foreignColumn": "basketId" + }, + { + "tableName": "outputs", + "name": "outputs_spentby_foreign", + "position": 1, + "columnName": "spentBy", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_foreign", + "position": 1, + "columnName": "transactionId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "outputs", + "name": "outputs_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "proven_txs", + "foreignColumn": "provenTxId" + }, + { + "tableName": "sync_states", + "name": "sync_states_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "transactions", + "name": "transactions_proventxid_foreign", + "position": 1, + "columnName": "provenTxId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "proven_txs", + "foreignColumn": "provenTxId" + }, + { + "tableName": "transactions", + "name": "transactions_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "tx_labels", + "name": "tx_labels_userid_foreign", + "position": 1, + "columnName": "userId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "users", + "foreignColumn": "userId" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_transactionid_foreign", + "position": 1, + "columnName": "transactionId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "transactions", + "foreignColumn": "transactionId" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_foreign", + "position": 1, + "columnName": "txLabelId", + "foreignSchema": "ts569_snapshot", + "foreignTable": "tx_labels", + "foreignColumn": "txLabelId" + } + ], + "foreignRules": [ + { + "tableName": "certificate_fields", + "name": "certificate_fields_certificateid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "certificate_fields", + "name": "certificate_fields_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "certificates", + "name": "certificates_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "commissions", + "name": "commissions_transactionid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "commissions", + "name": "commissions_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_baskets", + "name": "output_baskets_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_tags", + "name": "output_tags_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "output_tags_map", + "name": "output_tags_map_outputtagid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_basketid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_spentby_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_transactionid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "outputs", + "name": "outputs_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "proven_tx_reqs", + "name": "proven_tx_reqs_proventxid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "sync_states", + "name": "sync_states_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "transactions", + "name": "transactions_proventxid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "transactions", + "name": "transactions_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "tx_labels", + "name": "tx_labels_userid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_transactionid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + }, + { + "tableName": "tx_labels_map", + "name": "tx_labels_map_txlabelid_foreign", + "matchOption": "NONE", + "updateRule": "NO ACTION", + "deleteRule": "NO ACTION" + } + ], + "checks": [], + "triggers": [ + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (CAST(OLD.fieldName AS BINARY) <=> CAST(NEW.fieldName AS BINARY)) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_certificate_field_keys WHERE snapshotFieldName = OLD.fieldName AND snapshotCertificateId = OLD.certificateId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificate_fields", + "name": "snapshot_certificate_field_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) VALUES(NEW.userId, NEW.fieldName, NEW.certificateId, 1) ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 1; INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT userId, NEW.fieldName, NEW.certificateId, 2 FROM certificates WHERE certificateId = NEW.certificateId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_certificate_field_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId; DELETE FROM snapshot_certificate_field_keys WHERE snapshotUserId = OLD.userId AND snapshotCertificateId = OLD.certificateId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_certificate_parent_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_certificate_field_keys (snapshotUserId, snapshotFieldName, snapshotCertificateId, snapshotMembership) SELECT NEW.userId, f.fieldName, f.certificateId, 2 FROM snapshot_certificate_field_keys k JOIN certificate_fields f ON f.fieldName = k.snapshotFieldName AND f.certificateId = k.snapshotCertificateId WHERE k.snapshotCertificateId = NEW.certificateId AND (k.snapshotMembership & 1) = 1 ORDER BY k.snapshotFieldName, k.snapshotUserId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshot_certificate_field_keys.snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_profile_2_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_profile_2_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "certificates", + "name": "snapshot_profile_2_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.certificateId <=> NEW.certificateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 2 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.certificateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (2, NEW.userId, NEW.certificateId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "commissions", + "name": "snapshot_profile_6_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "commissions", + "name": "snapshot_profile_6_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "commissions", + "name": "snapshot_profile_6_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.commissionId <=> NEW.commissionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 6 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.commissionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (6, NEW.userId, NEW.commissionId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_baskets", + "name": "snapshot_profile_4_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_baskets", + "name": "snapshot_profile_4_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_baskets", + "name": "snapshot_profile_4_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.basketId <=> NEW.basketId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 4 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.basketId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (4, NEW.userId, NEW.basketId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_profile_5_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_profile_5_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_profile_5_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputTagId <=> NEW.outputTagId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 5 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputTagId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (5, NEW.userId, NEW.outputTagId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.outputTagId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags", + "name": "snapshot_relation_1_left_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 1 FROM output_tags_map WHERE outputTagId = NEW.outputTagId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputTagId <=> NEW.outputTagId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotLeftId = OLD.outputTagId AND snapshotRightId = OLD.outputId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "output_tags_map", + "name": "snapshot_relation_1_map_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 1 FROM output_tags WHERE outputTagId = NEW.outputTagId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, userId, NEW.outputTagId, NEW.outputId, 2 FROM outputs WHERE outputId = NEW.outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_profile_1_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_profile_1_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_profile_1_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.outputId <=> NEW.outputId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.outputId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (1, NEW.userId, NEW.outputId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.outputId <=> NEW.outputId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 1 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.outputId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "outputs", + "name": "snapshot_relation_1_right_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 1, NEW.userId, outputTagId, outputId, 2 FROM output_tags_map WHERE outputId = NEW.outputId ORDER BY outputTagId, outputId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxReqId <=> NEW.provenTxReqId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE requestId=OLD.provenTxReqId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_tx_reqs", + "name": "snapshot_global_req_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; END IF; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,0,NEW.provenTxReqId,userId FROM transactions WHERE txid=NEW.txid ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) SELECT transactionId,NEW.provenTxReqId,1,NEW.provenTxId,userId FROM transactions WHERE txid=NEW.txid AND NEW.provenTxId IS NOT NULL ORDER BY transactionId FOR SHARE ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.provenTxId <=> NEW.provenTxId) THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (OLD.provenTxId,0) ON DUPLICATE KEY UPDATE present=0; UPDATE snapshot_global_keys SET present=0 WHERE tableId=1 AND rowId=OLD.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=OLD.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.provenTxId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "proven_txs", + "name": "snapshot_global_proof_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,1) ON DUPLICATE KEY UPDATE present=1; UPDATE snapshot_global_keys SET present=1 WHERE tableId=1 AND rowId=NEW.provenTxId; DELETE FROM snapshot_global_guards WHERE proofId=NEW.provenTxId AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=NEW.provenTxId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_edge_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_global_keys SET refs=refs-1 WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId; DELETE FROM snapshot_global_keys WHERE tableId=OLD.tableId AND userId=OLD.userId AND rowId=OLD.rowId AND refs=0; DELETE FROM snapshot_global_guards WHERE proofId=OLD.rowId AND OLD.tableId=1 AND NOT EXISTS(SELECT 1 FROM snapshot_global_keys WHERE tableId=1 AND rowId=OLD.rowId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "snapshot_global_edges", + "name": "snapshot_global_edge_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_global_keys (tableId,userId,rowId,refs,present) VALUES (NEW.tableId,NEW.userId,NEW.rowId,1,CASE WHEN NEW.tableId=0 THEN 1 ELSE (SELECT present FROM snapshot_global_guards WHERE proofId=NEW.rowId FOR SHARE) END) ON DUPLICATE KEY UPDATE refs=snapshot_global_keys.refs+1; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "sync_states", + "name": "snapshot_profile_7_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "sync_states", + "name": "snapshot_profile_7_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "sync_states", + "name": "snapshot_profile_7_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.syncStateId <=> NEW.syncStateId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 7 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.syncStateId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (7, NEW.userId, NEW.syncStateId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 3, + "definer": "root@%", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txid <=> NEW.txid) OR NOT (OLD.provenTxId <=> NEW.provenTxId) THEN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 3, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_global_edges WHERE transactionId=OLD.transactionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_global_tx_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 3, + "definer": "root@%", + "body": "BEGIN DECLARE requestedId INT UNSIGNED DEFAULT NULL; DECLARE requestedProof INT UNSIGNED DEFAULT NULL; DECLARE CONTINUE HANDLER FOR NOT FOUND BEGIN SET requestedId=NULL; SET requestedProof=NULL; END; IF NEW.provenTxId IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (NEW.provenTxId,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=NEW.provenTxId; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,0,1,NEW.provenTxId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; SELECT provenTxReqId,provenTxId INTO requestedId,requestedProof FROM proven_tx_reqs WHERE txid=NEW.txid FOR SHARE; IF requestedId IS NOT NULL THEN INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,0,requestedId,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; IF requestedProof IS NOT NULL THEN INSERT INTO snapshot_global_guards (proofId,present) VALUES (requestedProof,0) ON DUPLICATE KEY UPDATE proofId=snapshot_global_guards.proofId; UPDATE snapshot_global_guards g LEFT JOIN proven_txs p ON p.provenTxId=g.proofId SET g.present=(p.provenTxId IS NOT NULL) WHERE g.proofId=requestedProof; INSERT INTO snapshot_global_edges (transactionId,requestId,tableId,rowId,userId) VALUES (NEW.transactionId,requestedId,1,requestedProof,NEW.userId) ON DUPLICATE KEY UPDATE transactionId = snapshot_global_edges.transactionId; END IF; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_profile_0_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_profile_0_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_profile_0_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.transactionId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (0, NEW.userId, NEW.transactionId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.transactionId <=> NEW.transactionId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 1 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotRightId = OLD.transactionId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "transactions", + "name": "snapshot_relation_0_right_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 2 FROM tx_labels_map WHERE transactionId = NEW.transactionId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_profile_3_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_profile_3_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_profile_3_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.userId <=> NEW.userId) OR NOT (OLD.txLabelId <=> NEW.txLabelId) THEN DELETE FROM snapshot_profile_keys WHERE snapshotTableId = 3 AND snapshotUserId = OLD.userId AND snapshotRowId = OLD.txLabelId; INSERT INTO snapshot_profile_keys (snapshotTableId, snapshotUserId, snapshotRowId) VALUES (3, NEW.userId, NEW.txLabelId); END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.userId <=> NEW.userId) THEN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN UPDATE snapshot_relation_keys SET snapshotMembership = snapshotMembership & 2 WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId; DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotUserId = OLD.userId AND snapshotLeftId = OLD.txLabelId AND snapshotMembership = 0; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels", + "name": "snapshot_relation_0_left_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 2, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, NEW.userId, txLabelId, transactionId, 1 FROM tx_labels_map WHERE txLabelId = NEW.txLabelId ORDER BY txLabelId, transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_after_update", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_before_update", + "event": "UPDATE", + "timing": "BEFORE", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.txLabelId <=> NEW.txLabelId) OR NOT (OLD.transactionId <=> NEW.transactionId) THEN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_delete", + "event": "DELETE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN DELETE FROM snapshot_relation_keys WHERE snapshotTableId = 0 AND snapshotLeftId = OLD.txLabelId AND snapshotRightId = OLD.transactionId; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "tx_labels_map", + "name": "snapshot_relation_0_map_insert", + "event": "INSERT", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 1 FROM tx_labels WHERE txLabelId = NEW.txLabelId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 1; INSERT INTO snapshot_relation_keys (snapshotTableId, snapshotUserId, snapshotLeftId, snapshotRightId, snapshotMembership) SELECT 0, userId, NEW.txLabelId, NEW.transactionId, 2 FROM transactions WHERE transactionId = NEW.transactionId FOR SHARE ON DUPLICATE KEY UPDATE snapshotMembership = snapshotMembership | 2; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + }, + { + "tableName": "users", + "name": "snapshot_sync_primary_change", + "event": "UPDATE", + "timing": "AFTER", + "actionOrder": 1, + "definer": "root@%", + "body": "BEGIN IF NOT (OLD.activeStorage <=> NEW.activeStorage) THEN\n INSERT INTO snapshot_sync_primary_epochs (userId, epoch) VALUES (NEW.userId, 1)\n ON DUPLICATE KEY UPDATE epoch = epoch + 1;\n END IF; END", + "sqlMode": "IGNORE_SPACE,ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_ENGINE_SUBSTITUTION", + "charset": "utf8mb4", + "connectionCollation": "utf8mb4_unicode_ci", + "databaseCollation": "utf8mb4_0900_ai_ci" + } + ] +} diff --git a/packages/wallet/wallet-toolbox/test/storage/portable.test.ts b/packages/wallet/wallet-toolbox/test/storage/portable.test.ts index 881d85239..ac1103c09 100644 --- a/packages/wallet/wallet-toolbox/test/storage/portable.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/portable.test.ts @@ -113,6 +113,85 @@ describe('BRC-38/39 portable wallet data', () => { await expect(importBRC38(target, document, { mode: 'restore' })).rejects.toThrow(/empty target storage/) }) + test('canonical capture omits absent legacy JSON object fields without changing source history or array positions', async () => { + const source = await createPortableSource('portable_nullable_history', '4'.repeat(64)) + const storage = source.activeStorage + const original = await exportBRC38(storage, source.identityKey) + const partial = { provenTxReqId: original.tables.provenTxReqs[0].provenTxReqId as number } + const request = verifyOne(await storage.findProvenTxReqs({ partial })) + const history = JSON.stringify({ + notes: [{ when: null, what: 'sent', optional: null, retained: false, zero: 0, text: '', ['__proto__']: 'own JSON property' }] + }) + await storage.updateProvenTxReq(request.provenTxReqId, { history }) + const document = await exportBRC38(storage, source.identityKey) + expect(document.tables.provenTxReqs[0].history).toEqual({ + notes: [{ what: 'sent', retained: false, zero: 0, text: '', ['__proto__']: 'own JSON property' }] + }) + expect(verifyOne(await storage.findProvenTxReqs({ partial })).history).toBe(history) + expect(parseBRC38Json(await exportBRC38Json(storage, source.identityKey)).tables.provenTxReqs[0].history) + .toEqual(document.tables.provenTxReqs[0].history) + }) + + test('canonical capture refuses null required history fields instead of hiding them', async () => { + const source = await createPortableSource('portable_null_required', '6'.repeat(64)) + const original = await exportBRC38(source.activeStorage, source.identityKey) + const provenTxReqId = original.tables.provenTxReqs[0].provenTxReqId as number + await source.activeStorage.updateProvenTxReq(provenTxReqId, { history: '{"notes":[{"what":null}]}' }) + await expect(exportBRC38(source.activeStorage, source.identityKey)).rejects.toThrow('must not contain null') + }) + + test.each(['labels', 'tags'])('capture refuses inconsistent %s relationships instead of omitting owned map rows', async kind => { + const source = await createPortableSource(`portable_inconsistent_${kind}`, '8'.repeat(64)) + const setup = source.setup! + if (kind === 'labels') { + await _tu.insertTestTxLabelMap(source.activeStorage, setup.u2tx1, setup.u1label1) + await expect(exportBRC38(source.activeStorage, source.identityKey)).rejects.toThrow('txLabelMap.transactionId') + expect(await source.activeStorage.findTxLabelMaps({ partial: { + transactionId: setup.u2tx1.transactionId, txLabelId: setup.u1label1.txLabelId + } })).toHaveLength(1) + } else { + await _tu.insertTestOutputTagMap(source.activeStorage, setup.u2tx1o0, setup.u1tag1) + await expect(exportBRC38(source.activeStorage, source.identityKey)).rejects.toThrow('outputTagMap.outputId') + expect(await source.activeStorage.findOutputTagMaps({ partial: { + outputId: setup.u2tx1o0.outputId, outputTagId: setup.u1tag1.outputTagId + } })).toHaveLength(1) + } + }) + + test('legacy optional sync/error fields normalize while checkpoint counts and ID mappings remain authoritative', async () => { + const source = await createPortableSource('portable_nullable_checkpoint', '7'.repeat(64)) + const storage = source.activeStorage + const original = await exportBRC38(storage, source.identityKey) + const originalState = verifyTruthy(original.tables.syncStates.find(row => row.storageIdentityKey === remoteSyncStorageIdentityKey)) + const partial = { syncStateId: originalState.syncStateId as number } + const state = verifyOne(await storage.findSyncStates({ partial })) + const map = JSON.parse(state.syncMap) + map.transaction.maxUpdated_at = null + const syncMap = JSON.stringify(map) + const errorLocal = '{"code":"retry","description":"synthetic checkpoint","stack":null}' + await storage.updateSyncState(state.syncStateId, { syncMap, errorLocal }) + const captured = await exportBRC38(storage, source.identityKey) + const row = verifyTruthy(captured.tables.syncStates.find(row => row.syncStateId === state.syncStateId)) + expect(row.errorLocal).toEqual({ code: 'retry', description: 'synthetic checkpoint' }) + delete map.transaction.maxUpdated_at + expect(row.syncMap).toEqual(map) + expect(verifyOne(await storage.findSyncStates({ partial })).syncMap).toBe(syncMap) + map.transaction.idMap['999'] = null + await storage.updateSyncState(state.syncStateId, { syncMap: JSON.stringify(map) }) + await expect(exportBRC38(storage, source.identityKey)).rejects.toThrow('must not contain null') + }) + + test('canonical capture rejects a null array entry instead of silently dropping it', async () => { + const source = await createPortableSource('portable_null_array', '5'.repeat(64)) + const original = await exportBRC38(source.activeStorage, source.identityKey) + const partial = { provenTxReqId: original.tables.provenTxReqs[0].provenTxReqId as number } + const request = verifyOne(await source.activeStorage.findProvenTxReqs({ partial })) + const history = '{"notes":[{"what":"before"},null,{"what":"after"}]}' + await source.activeStorage.updateProvenTxReq(request.provenTxReqId, { history }) + await expect(exportBRC38(source.activeStorage, source.identityKey)).rejects.toThrow('must not contain null') + expect(verifyOne(await source.activeStorage.findProvenTxReqs({ partial })).history).toBe(history) + }) + test('normalizes an exact legacy managed-change default during BRC-38 restore', async () => { const document = minimalDocument() document.tables.outputBaskets.push( diff --git a/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts b/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts index 09556bb4d..38c110f9a 100644 --- a/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts +++ b/packages/wallet/wallet-toolbox/test/storage/processAction.test.ts @@ -3,7 +3,7 @@ import { processAction, shareReqsWithWorld } from '../../src/storage/methods/pro import { StorageProvider } from '../../src/storage/StorageProvider' import { TableProvenTxReq } from '../../src/storage/schema/tables/TableProvenTxReq' -function makeReadyReq (): TableProvenTxReq { +function makeReadyReq(): TableProvenTxReq { const now = new Date() return { created_at: now, @@ -20,7 +20,7 @@ function makeReadyReq (): TableProvenTxReq { } } -function makeStorageFake () { +function makeStorageFake() { return { transaction: jest.fn(async (callback: (trx?: unknown) => Promise) => await callback(undefined)), updateProvenTxReq: jest.fn(async () => 1), @@ -42,23 +42,29 @@ describe('processAction shareReqsWithWorld', () => { telemetry: new Telemetry({ sink: { capture: event => events.push(event) } }) } - const result = await processAction(storage as any, { userId: 1 }, { - isNewTx: false, - isSendWith: false, - isNoSend: true, - isDelayed: false, - sendWith: [] - }) + const result = await processAction( + storage as any, + { userId: 1 }, + { + isNewTx: false, + isSendWith: false, + isNoSend: true, + isDelayed: false, + sendWith: [] + } + ) expect(result.sendWithResults).toEqual([]) - expect(events).toEqual(expect.arrayContaining([ - expect.objectContaining({ name: 'wallet.storage.process_action.share', spanStatus: 'ok' }), - expect.objectContaining({ - name: 'wallet.storage.process_action', - spanStatus: 'ok', - attributes: expect.objectContaining({ 'action.send_result_count': 0 }) - }) - ])) + expect(events).toEqual( + expect.arrayContaining([ + expect.objectContaining({ name: 'wallet.storage.process_action.share', spanStatus: 'ok' }), + expect.objectContaining({ + name: 'wallet.storage.process_action', + spanStatus: 'ok', + attributes: expect.objectContaining({ 'action.send_result_count': 0 }) + }) + ]) + ) }) test('preserves the non-instrumented processAction path when telemetry is disabled', async () => { @@ -67,13 +73,19 @@ describe('processAction shareReqsWithWorld', () => { telemetry: new Telemetry() } - await expect(processAction(storage as any, { userId: 1 }, { - isNewTx: false, - isSendWith: false, - isNoSend: true, - isDelayed: false, - sendWith: [] - })).resolves.toMatchObject({ sendWithResults: [] }) + await expect( + processAction( + storage as any, + { userId: 1 }, + { + isNewTx: false, + isSendWith: false, + isNoSend: true, + isDelayed: false, + sendWith: [] + } + ) + ).resolves.toMatchObject({ sendWithResults: [] }) }) test('delayed sends do not build aggregate BEEF before scheduling', async () => { @@ -90,7 +102,11 @@ describe('processAction shareReqsWithWorld', () => { const result = await shareReqsWithWorld(storage as any, 1, [req.txid], true) expect(storage.getReqsAndBeefToShareWithWorld).not.toHaveBeenCalled() - expect(storage.updateProvenTxReq).toHaveBeenCalledWith([req.provenTxReqId], expect.objectContaining({ status: 'unsent' }), undefined) + expect(storage.updateProvenTxReq).toHaveBeenCalledWith( + [req.provenTxReqId], + expect.objectContaining({ status: 'unsent' }), + undefined + ) expect(storage.updateTransaction).toHaveBeenCalledWith([22], { status: 'sending' }, undefined) expect(result.swr).toEqual([{ txid: req.txid, status: 'sending' }]) }) @@ -110,7 +126,11 @@ describe('processAction shareReqsWithWorld', () => { }) expect(beef.verify).not.toHaveBeenCalled() - expect(storage.updateProvenTxReq).toHaveBeenCalledWith([req.provenTxReqId], expect.objectContaining({ status: 'unsent' }), undefined) + expect(storage.updateProvenTxReq).toHaveBeenCalledWith( + [req.provenTxReqId], + expect.objectContaining({ status: 'unsent' }), + undefined + ) expect(storage.updateTransaction).toHaveBeenCalledWith([22], { status: 'sending' }, undefined) expect(result.swr).toEqual([{ txid: req.txid, status: 'sending' }]) }) @@ -176,23 +196,24 @@ describe('processAction shareReqsWithWorld', () => { const result = await storage.getReqsAndBeefToShareWithWorld([req.txid], []) - expect(result.details).toEqual([ - expect.objectContaining({ txid: req.txid, status: 'error' }) - ]) + expect(result.details).toEqual([expect.objectContaining({ txid: req.txid, status: 'error' })]) }) test('immediate sends still validate the aggregate BEEF before broadcasting', async () => { const req = makeReadyReq() const beef = { + bumps: [], verify: jest.fn(async () => false), toLogString: () => 'invalid beef' } as unknown as Beef const storage = makeStorageFake() - await expect(shareReqsWithWorld(storage as any, 1, [req.txid], false, { - beef, - details: [{ txid: req.txid, status: 'readyToSend', req }] - })).rejects.toThrow('merged Beef failed validation') + await expect( + shareReqsWithWorld(storage as any, 1, [req.txid], false, { + beef, + details: [{ txid: req.txid, status: 'readyToSend', req }] + }) + ).rejects.toThrow('merged Beef failed validation') expect(beef.verify).toHaveBeenCalled() expect(storage.attemptToPostReqsToNetwork).not.toHaveBeenCalled() diff --git a/packages/wallet/wallet-toolbox/test/storage/runSnapshotArchiveMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/runSnapshotArchiveMysql.cjs new file mode 100644 index 000000000..8dfba7ef6 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/runSnapshotArchiveMysql.cjs @@ -0,0 +1,154 @@ +// Disposable, loopback-only synthetic MySQL qualification. Never targets an +// operator-supplied database, retains a volume, pulls an image, or builds one. +const { execFile } = require('node:child_process') +const { randomBytes, randomUUID } = require('node:crypto') +const { join } = require('node:path') +const assert = require('node:assert/strict') +const { executable, context, image, validateContext, validateContainer } = require('./snapshotArchiveDocker.cjs') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { mysqlFixtureGroups } = require('./snapshotMysqlFixtureGroups.cjs') +const execute = (file, args, options) => + new Promise((resolve, reject) => { + execFile( + file, + args, + { encoding: 'utf8', killSignal: 'SIGKILL', maxBuffer: 1048576, ...options }, + (error, stdout) => { + if (error) reject(error) + else resolve(stdout) + } + ) + }) + +async function main() { + const secret = randomBytes(32).toString('hex') + const fixtureEnvironment = { ...process.env, MYSQL_ROOT_PASSWORD: secret, MYSQL_PWD: secret } + const cancellation = new AbortController() + const interrupt = () => cancellation.abort(new Error('Snapshot fixture cancelled')) + process.once('SIGINT', interrupt) + process.once('SIGTERM', interrupt) + const command = async (args, signal) => + ( + await execute(executable, ['--context', context, ...args], { + timeout: 15000, + env: fixtureEnvironment, + ...(signal === undefined ? {} : { signal }) + }) + ).trim() + const docker = (...args) => command(args, cancellation.signal) + const cleanup = (...args) => command(args) + const owner = randomUUID() + const name = 'ts569-durable-' + owner + let id + let creating = false + let failure + try { + validateContext(JSON.parse(await docker('context', 'inspect', context))) + await docker('image', 'inspect', image) + creating = true + id = await docker( + 'run', + '--pull=never', + '--detach', + '--name', + name, + '--label', + 'network-ops.fixture=ts-stack-544-durable', + '--label', + 'network-ops.fixture-owner=' + owner, + '--memory', + '1g', + '--cpus', + '2', + '--pids-limit', + '256', + '--tmpfs', + '/var/lib/mysql:rw,nosuid,nodev,size=512m', + '--env', + 'MYSQL_ROOT_PASSWORD', + '--env', + 'MYSQL_DATABASE=ts569_snapshot', + '--publish', + '127.0.0.1::3306', + image + ) + validateContainer(JSON.parse(await docker('inspect', id))[0], { name, owner, id }) + const deadline = Date.now() + 60000 + let ready = false + function* pendingReadiness() { + while (!ready) yield undefined + } + await runInSeries(pendingReadiness(), async () => { + try { + // TCP specifically excludes the entrypoint's temporary socket-only server. + await docker( + 'exec', + '--env', + 'MYSQL_PWD', + id, + 'mysqladmin', + '--protocol=tcp', + '--host=127.0.0.1', + '--user=root', + 'ping' + ) + ready = true + } catch (error) { + if (cancellation.signal.aborted || Date.now() >= deadline) throw error + await new Promise(resolve => setTimeout(resolve, 500)) + } + }) + await runInSeries(mysqlFixtureGroups, async group => { + const started = Date.now() + process.stdout.write(JSON.stringify({ group, status: 'started' }) + '\n') + const result = await execute(process.execPath, [join(__dirname, 'snapshotArchiveMysql.cjs'), group], { + env: { + ...process.env, + TS_STACK_SNAPSHOT_CONTAINER: name, + TS_STACK_SNAPSHOT_CONTAINER_ID: id, + TS_STACK_SNAPSHOT_CONTAINER_OWNER: owner, + TS_STACK_SNAPSHOT_MYSQL_SECRET: secret + }, + signal: cancellation.signal, + timeout: 60000 + }) + process.stdout.write(result) + process.stdout.write(JSON.stringify({ group, status: 'passed', milliseconds: Date.now() - started }) + '\n') + }) + } catch (error) { + failure = error + } + let cleanupFailure + if (creating) { + // A timed-out create may have succeeded before its reply was lost. Cleanup + // keeps its own deadline after cancellation and requires the complete claim. + try { + const owned = () => cleanup('ps', '-aq', '--filter', 'label=network-ops.fixture-owner=' + owner) + const pending = await owned() + if (pending !== '') { + const containers = JSON.parse(await cleanup('inspect', ...pending.split('\n'))) + assert.equal(containers.length, 1) + validateContainer(containers[0], { name, owner, id }) + await cleanup('rm', '--force', containers[0].Id) + } + assert.equal(await owned(), '') + } catch (error) { + cleanupFailure = error + } + } + process.removeListener('SIGINT', interrupt) + process.removeListener('SIGTERM', interrupt) + if (cleanupFailure !== undefined) + throw new AggregateError( + failure === undefined ? [cleanupFailure] : [failure, cleanupFailure], + 'Snapshot fixture cleanup is unproved' + ) + if (failure !== undefined) throw failure + cancellation.signal.throwIfAborted() +} +module.exports = main +if (require.main === module) + main().catch(error => { + console.error(error) + process.exitCode = 1 + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/runSnapshotJournalMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/runSnapshotJournalMysql.cjs new file mode 100644 index 000000000..70a1d2bad --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/runSnapshotJournalMysql.cjs @@ -0,0 +1,193 @@ +// Disposable, loopback-only synthetic MySQL qualification. Never targets an +// operator-supplied database, retains a volume, pulls an image, or builds one. +const { execFile } = require('node:child_process') +const { randomBytes, randomUUID } = require('node:crypto') +const { join } = require('node:path') +const assert = require('node:assert/strict') +const { executable, context, image, validateContext, validateContainer } = require('./snapshotArchiveDocker.cjs') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const journalFixtureGroups = Object.freeze([ + 'generation', + 'server-crash', + 'receipts', + 'capture', + 'retention-rc', + 'retention-rr', + 'maintenance-rc', + 'maintenance-rr' +]) +const execute = (file, args, options) => + new Promise((resolve, reject) => { + execFile( + file, + args, + { encoding: 'utf8', killSignal: 'SIGKILL', maxBuffer: 1048576, ...options }, + (error, stdout) => { + if (error) reject(error) + else resolve(stdout) + } + ) + }) + +function fixtureScript(group) { + if (group === 'maintenance-rc') return 'snapshotJournalMaintenanceMysqlRc.cjs' + if (group === 'maintenance-rr') return 'snapshotJournalMaintenanceMysqlRr.cjs' + if (group === 'retention-rc') return 'snapshotJournalRetentionMysqlRc.cjs' + if (group === 'retention-rr') return 'snapshotJournalRetentionMysqlRr.cjs' + if (group === 'capture') return 'snapshotJournalCaptureMysql.cjs' + if (group === 'generation') return 'snapshotJournalMysql.cjs' + if (group === 'server-crash') return 'snapshotJournalMysqlServerCrash.cjs' + return 'snapshotJournalReceiptMysql.cjs' +} +function fixtureTimeout(group) { + if ( + group === 'generation' || + group === 'capture' || + group.startsWith('retention-') || + group.startsWith('maintenance-') + ) + return 180000 + if (group === 'server-crash') return 240000 + return 60000 +} + +async function runFixture(group) { + assert(journalFixtureGroups.includes(group)) + const secret = randomBytes(32).toString('hex') + const fixtureEnvironment = { ...process.env, MYSQL_ROOT_PASSWORD: secret, MYSQL_PWD: secret } + const cancellation = new AbortController() + const interrupt = () => cancellation.abort(new Error('Snapshot fixture cancelled')) + process.once('SIGINT', interrupt) + process.once('SIGTERM', interrupt) + const command = async (args, signal) => + ( + await execute(executable, ['--context', context, ...args], { + timeout: 15000, + env: fixtureEnvironment, + ...(signal === undefined ? {} : { signal }) + }) + ).trim() + const docker = (...args) => command(args, cancellation.signal) + const cleanup = (...args) => command(args) + const owner = randomUUID() + const name = 'ts569-durable-' + owner + let id + let creating = false + let failure + try { + validateContext(JSON.parse(await docker('context', 'inspect', context))) + await docker('image', 'inspect', image) + creating = true + id = await docker( + 'run', + '--pull=never', + '--detach', + '--name', + name, + '--label', + 'network-ops.fixture=ts-stack-544-durable', + '--label', + 'network-ops.fixture-owner=' + owner, + '--memory', + '1g', + '--cpus', + '2', + '--pids-limit', + '256', + '--tmpfs', + '/var/lib/mysql:rw,nosuid,nodev,size=512m', + '--env', + 'MYSQL_ROOT_PASSWORD', + '--env', + 'MYSQL_DATABASE=ts569_snapshot', + '--publish', + '127.0.0.1::3306', + '--entrypoint', + '/bin/sh', + image, + '-c', + 'attempt=0; while [ "$attempt" -lt 32 ]; do attempt=$((attempt + 1)); /usr/local/bin/docker-entrypoint.sh mysqld --pid-file=/var/lib/mysql/fixture.pid & wait "$!"; done; exit 1' + ) + validateContainer(JSON.parse(await docker('inspect', id))[0], { name, owner, id }) + const deadline = Date.now() + 60000 + let ready = false + function* pendingReadiness() { + while (!ready) yield undefined + } + await runInSeries(pendingReadiness(), async () => { + try { + // TCP specifically excludes the entrypoint's temporary socket-only server. + await docker( + 'exec', + '--env', + 'MYSQL_PWD', + id, + 'mysqladmin', + '--protocol=tcp', + '--host=127.0.0.1', + '--user=root', + 'ping' + ) + ready = true + } catch (error) { + if (cancellation.signal.aborted || Date.now() >= deadline) throw error + await new Promise(resolve => setTimeout(resolve, 500)) + } + }) + { + const started = Date.now() + process.stdout.write(JSON.stringify({ group, status: 'started' }) + '\n') + const result = await execute(process.execPath, [join(__dirname, fixtureScript(group)), group], { + env: { + ...process.env, + TS_STACK_SNAPSHOT_CONTAINER: name, + TS_STACK_SNAPSHOT_CONTAINER_ID: id, + TS_STACK_SNAPSHOT_CONTAINER_OWNER: owner, + TS_STACK_SNAPSHOT_MYSQL_SECRET: secret + }, + signal: cancellation.signal, + timeout: fixtureTimeout(group) + }) + process.stdout.write(result) + process.stdout.write(JSON.stringify({ group, status: 'passed', milliseconds: Date.now() - started }) + '\n') + } + } catch (error) { + failure = error + } + let cleanupFailure + if (creating) { + // A timed-out create may have succeeded before its reply was lost. Cleanup + // keeps its own deadline after cancellation and requires the complete claim. + try { + const owned = () => cleanup('ps', '-aq', '--filter', 'label=network-ops.fixture-owner=' + owner) + const pending = await owned() + if (pending !== '') { + const containers = JSON.parse(await cleanup('inspect', ...pending.split('\n'))) + assert.equal(containers.length, 1) + validateContainer(containers[0], { name, owner, id }) + await cleanup('rm', '--force', containers[0].Id) + } + assert.equal(await owned(), '') + } catch (error) { + cleanupFailure = error + } + } + process.removeListener('SIGINT', interrupt) + process.removeListener('SIGTERM', interrupt) + if (cleanupFailure !== undefined) + throw new AggregateError( + failure === undefined ? [cleanupFailure] : [failure, cleanupFailure], + 'Snapshot fixture cleanup is unproved' + ) + if (failure !== undefined) throw failure + cancellation.signal.throwIfAborted() +} +async function main() { + await runInSeries(journalFixtureGroups, runFixture) +} +module.exports = main +if (require.main === module) + main().catch(error => { + console.error(error) + process.exitCode = 1 + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveCrash.cjs new file mode 100644 index 000000000..bdcf0c946 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveCrash.cjs @@ -0,0 +1,181 @@ +// Synthetic SQLite process-termination qualification. Run after pnpm build. +// Each child owns a fresh temporary database; no wallet or service is contacted. +const assert = require('node:assert/strict') +const fs = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { spawn } = require('node:child_process') +const { knex } = require('knex') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + KnexSnapshotArchiveStore, + snapshotArchiveTables +} = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveStore.js') +const { addSnapshotArchiveTables } = require('../../out/src/storage/schema/snapshotArchiveMigration.js') +const identity = '02' + '11'.repeat(32), + date = new Date('2026-01-01T00:00:00.000Z') +const binding = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceSchema: 'synthetic-v1', + sourceStorage: { + created_at: date, + updated_at: date, + storageIdentityKey: 'source', + storageName: 'source', + chain: 'test', + dbtype: 'SQLite', + maxOutputScript: 1024 + }, + user: { created_at: date, updated_at: date, userId: 7, identityKey: identity, activeStorage: 'source' } +} +const phases = [ + 'after-begin', + 'after-page-insert', + 'after-page-checkpoint', + 'after-append-commit', + 'after-seal-before-ack' +] +const open = directory => + knex({ + client: 'better-sqlite3', + connection: { filename: path.join(directory, 'archive.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) +async function child(directory, phase) { + assert(phases.includes(phase)) + const db = open(directory) + await db.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(db) + const store = new KnexSnapshotArchiveStore(db) + const writer = await store.begin(binding) + fs.writeFileSync(path.join(directory, 'writer.json'), JSON.stringify(writer), { mode: 0o600 }) + const park = () => { + fs.writeFileSync(path.join(directory, 'boundary'), phase) + Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0) + } + if (phase === 'after-begin') park() + db.on('query-response', (_result, query) => { + if (phase === 'after-page-insert' && query.sql.startsWith('insert into `snapshot_archive_pages`')) park() + if ( + phase === 'after-page-checkpoint' && + query.sql.startsWith('update `snapshot_archives`') && + query.sql.includes('`nextSequence`') + ) + park() + }) + await runInSeries(snapshotArchiveTables.entries(), async ([sequence, table]) => { + await store.append(writer, { sequence, table, rows: 0, done: true, bytes: new Uint8Array([91, 93]) }) + if (phase === 'after-append-commit') park() + }) + await store.seal(writer) + if (phase === 'after-seal-before-ack') park() + throw new Error('Expected boundary was not reached') +} +async function parent() { + await runInSeries(phases, async phase => { + const directory = fs.mkdtempSync(path.join(os.tmpdir(), 'ts569-archive-crash-')) + let processHandle, database, exited + try { + processHandle = spawn(process.execPath, [__filename, 'child', phase], { + cwd: directory, + env: process.env, + stdio: ['ignore', 'ignore', 'pipe', 'ipc'] + }) + let error = '' + processHandle.stderr.on('data', chunk => { + error = (error + chunk.toString()).slice(-65536) + }) + exited = new Promise((resolve, reject) => { + processHandle.once('exit', (code, signal) => resolve({ code, signal })) + processHandle.once('error', reject) + }) + const deadline = Date.now() + 15000 + let reachedBoundary = fs.existsSync(path.join(directory, 'boundary')) + function* pendingBoundary() { + while (!reachedBoundary) yield undefined + } + await runInSeries(pendingBoundary(), async () => { + if (processHandle.exitCode !== null || processHandle.signalCode !== null) + throw new Error(error || 'Child stopped before boundary') + assert(Date.now() < deadline, 'Boundary deadline: ' + phase) + await new Promise(resolve => setTimeout(resolve, 20)) + reachedBoundary = fs.existsSync(path.join(directory, 'boundary')) + }) + assert.equal(fs.readFileSync(path.join(directory, 'boundary'), 'utf8'), phase) + processHandle.kill('SIGKILL') + const outcome = await exited + assert.equal(outcome.signal, 'SIGKILL') + database = open(directory) + const store = new KnexSnapshotArchiveStore(database) + const writer = JSON.parse(fs.readFileSync(path.join(directory, 'writer.json'), 'utf8')) + const before = await database('snapshot_archives').where({ archiveId: writer.archiveId }).first() + const ready = phase === 'after-seal-before-ack' + if (ready) { + assert.equal((await store.inspect(identity, writer.archiveId)).pages, 13) + assert.equal((await store.read(identity, writer.archiveId, 12)).table, 'syncStates') + } else await assert.rejects(store.inspect(identity, writer.archiveId), /unavailable/) + let committed = 0 + if (phase === 'after-append-commit') committed = 1 + else if (ready) committed = 13 + assert.equal(before.nextSequence, committed) + assert.equal(Number((await database('snapshot_archive_pages').count({ count: '*' }))[0].count), committed) + if (phase === 'after-append-commit') { + await store.append(writer, { + sequence: 0, + table: 'provenTxs', + rows: 0, + done: true, + bytes: new Uint8Array([91, 93]) + }) + assert.equal((await database('snapshot_archives').first()).nextSequence, 1) + } + await database('snapshot_archives').where({ archiveId: writer.archiveId }).update({ expiresAt: 0 }) + await store.reap() + assert.equal(Number((await database('snapshot_archive_capacity').first()).reservedBytes), 0) + assert.equal(Number((await database('snapshot_archive_pages').count({ count: '*' }))[0].count), 0) + console.log( + JSON.stringify({ + phase, + actualSignal: outcome.signal, + committedPages: committed, + ready, + exactReplay: phase === 'after-append-commit', + expiredCleanupComplete: true + }) + ) + } finally { + if (processHandle?.exitCode === null && processHandle.signalCode === null) { + processHandle.kill('SIGKILL') + await exited + } + if (database) await database.destroy() + fs.rmSync(directory, { recursive: true, force: true }) + } + }) +} +async function main() { + if (process.argv[2] === 'child') { + assert.equal(typeof process.send, 'function', 'Child execution requires its fixture parent') + await child(process.cwd(), process.argv[3]) + } else { + await parent() + const { qualifySQLiteGuardProcessLoss } = require('./snapshotArchiveGuardCrash.cjs') + console.log(JSON.stringify({ ownerProcessLoss: await qualifySQLiteGuardProcessLoss() })) + const { qualifySQLiteProfileIndexProcessLoss } = require('./snapshotProfileIndexCrash.cjs') + console.log(JSON.stringify({ profileIndexProcessLoss: await qualifySQLiteProfileIndexProcessLoss() })) + const { qualifySQLiteRelationIndexProcessLoss } = require('./snapshotRelationIndexCrash.cjs') + console.log(JSON.stringify({ relationIndexProcessLoss: await qualifySQLiteRelationIndexProcessLoss() })) + const { qualifySQLiteGlobalIndexProcessLoss } = require('./snapshotGlobalIndexCrash.cjs') + console.log(JSON.stringify({ globalIndexProcessLoss: await qualifySQLiteGlobalIndexProcessLoss() })) + const { qualifySQLiteCertificateIndexProcessLoss } = require('./snapshotCertificateIndexCrash.cjs') + console.log(JSON.stringify({ certificateIndexProcessLoss: await qualifySQLiteCertificateIndexProcessLoss() })) + const { qualifySQLiteGenerationProcessLoss } = require('./snapshotSqliteGenerationCrash.cjs') + console.log(JSON.stringify({ sqliteGenerationProcessLoss: await qualifySQLiteGenerationProcessLoss() })) + } +} +main().catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveDocker.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveDocker.cjs new file mode 100644 index 000000000..b8cf6164f --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveDocker.cjs @@ -0,0 +1,67 @@ +const assert = require('node:assert/strict') +const { existsSync } = require('node:fs') + +// Hosted qualification is explicit and may use only the runner's local daemon. +const mode = process.env.TS_STACK_SNAPSHOT_HOSTED_MYSQL +if (mode !== undefined) { + assert.equal(mode, '1', 'Unsupported snapshot fixture Docker mode') + assert.equal(process.platform, 'linux', 'Hosted fixture requires Linux') + assert.equal(process.env.GITHUB_ACTIONS, 'true', 'Hosted fixture requires GitHub Actions') + assert.equal(process.env.DOCKER_HOST, undefined, 'An external Docker host is not supported') + assert.equal(process.env.DOCKER_CONTEXT, undefined, 'An external Docker context is not supported') +} +const context = mode === '1' ? 'default' : 'desktop-linux' +const image = 'mysql@sha256:0744ee5ef89ce6ccfa13de3e579fe6b9e27f93dd70da9c06d2c908b1b193fb8d' + +// Fixture executables use known installation paths rather than searching PATH. +const executable = [ + '/Applications/Docker.app/Contents/Resources/bin/docker', + '/usr/bin/docker', + '/usr/local/bin/docker', + '/opt/homebrew/bin/docker' +].find(candidate => existsSync(candidate)) + +if (executable === undefined) throw new Error('Install Docker at a supported local installation path') + +function validateContext(records) { + assert.equal(Array.isArray(records) && records.length, 1, 'Expected one local Docker context') + assert.equal(records[0].Name, context) + const host = records[0].Endpoints?.docker?.Host + if (mode === '1') assert.equal(host, 'unix:///var/run/docker.sock', 'Hosted fixture requires the local Unix socket') + else + assert.equal( + typeof host === 'string' && host.startsWith('unix:///'), + true, + 'Desktop fixture requires a local Unix socket' + ) +} + +function validateContainer(actual, expected) { + assert.match(expected.owner, /^[a-f0-9]{8}-[a-f0-9]{4}-4[a-f0-9]{3}-[89ab][a-f0-9]{3}-[a-f0-9]{12}$/) + assert.equal(expected.name, 'ts569-durable-' + expected.owner) + assert.equal(actual.Name, '/' + expected.name) + assert.equal(actual.Config.Labels['network-ops.fixture'], 'ts-stack-544-durable') + assert.equal(actual.Config.Labels['network-ops.fixture-owner'], expected.owner) + assert.equal(actual.Config.Image, image) + if (expected.id !== undefined) assert.equal(actual.Id, expected.id) + assert.match(actual.Id, /^[a-f0-9]{64}$/) +} + +module.exports = { executable, context, image, validateContext, validateContainer } + +// The launcher never pulls. Provision this exact image only in an explicit +// hosted-runner step, before the disposable fixture acquires any resources. +if (require.main === module) { + assert.deepEqual(process.argv.slice(2), ['provision-hosted-image']) + assert.equal(mode, '1', 'Image provisioning requires explicit hosted mode') + const { execFileSync } = require('node:child_process') + validateContext( + JSON.parse( + execFileSync(executable, ['--context', context, 'context', 'inspect', context], { + encoding: 'utf8', + timeout: 15000 + }) + ) + ) + execFileSync(executable, ['--context', context, 'pull', image], { timeout: 240000, stdio: 'inherit' }) +} diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveDocker.test.ts b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveDocker.test.ts new file mode 100644 index 000000000..d819a191e --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveDocker.test.ts @@ -0,0 +1,99 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { runInNewContext } from 'node:vm' + +interface FixtureDocker { + executable: string + context: string + image: string + validateContext: (records: unknown) => void + validateContainer: (actual: unknown, expected: unknown) => void +} +const source = readFileSync(join(__dirname, 'snapshotArchiveDocker.cjs'), 'utf8') +const desktop = '/Applications/Docker.app/Contents/Resources/bin/docker' +const hosted = { TS_STACK_SNAPSHOT_HOSTED_MYSQL: '1', GITHUB_ACTIONS: 'true' } +function load(platform = 'darwin', env: Record = {}, available = desktop): FixtureDocker { + const module = { exports: {} } + runInNewContext(source, { + module, + process: { platform, env }, + require: (name: string) => { + if (name === 'node:fs') return { existsSync: (candidate: string) => candidate === available } + if (name === 'node:assert/strict') return assert + throw new Error('Unexpected fixture dependency') + } + }) + return module.exports as FixtureDocker +} +const endpoint = (name: string, host: string) => [{ Name: name, Endpoints: { docker: { Host: host } } }] + +test('local fixture retains its explicit Desktop context and known installation path', () => { + const docker = load() + expect(docker.context).toBe('desktop-linux') + expect(docker.executable).toBe(desktop) + expect(() => docker.validateContext(endpoint('desktop-linux', 'unix:///Users/synthetic/docker.sock'))).not.toThrow() + expect(() => docker.validateContext(endpoint('desktop-linux', 'tcp://remote.example:2375'))).toThrow() + expect(() => docker.validateContext(endpoint('default', 'unix:///var/run/docker.sock'))).toThrow() + expect(() => load('darwin', {}, '/untrusted/path/docker')).toThrow('supported local installation path') +}) + +test('explicit hosted Linux mode accepts only the runner default Unix socket', () => { + const docker = load('linux', hosted, '/usr/bin/docker') + expect(docker.context).toBe('default') + expect(docker.executable).toBe('/usr/bin/docker') + expect(() => docker.validateContext(endpoint('default', 'unix:///var/run/docker.sock'))).not.toThrow() + for (const records of [ + [], + {}, + endpoint('default', 'unix:///tmp/another.sock'), + endpoint('default', 'tcp://remote.example:2375'), + endpoint('desktop-linux', 'unix:///var/run/docker.sock'), + [...endpoint('default', 'unix:///var/run/docker.sock'), ...endpoint('default', 'unix:///var/run/docker.sock')] + ]) + expect(() => docker.validateContext(records)).toThrow() +}) + +test.each([ + ['darwin', hosted], + ['linux', { ...hosted, GITHUB_ACTIONS: 'false' }], + ['linux', { ...hosted, TS_STACK_SNAPSHOT_HOSTED_MYSQL: 'arbitrary' }], + ['linux', { ...hosted, DOCKER_HOST: 'tcp://remote.example:2375' }], + ['linux', { ...hosted, DOCKER_CONTEXT: 'another' }] +] as Array<[string, Record]>)( + 'invalid hosted fixture mode refuses before Docker I/O (%s, %j)', + (platform, env) => { + expect(() => load(platform, env, '/usr/bin/docker')).toThrow() + } +) + +test('cleanup requires the exact invocation claim, pinned image and container identity', () => { + const docker = load() + const owner = '00000000-0000-4000-8000-000000000001' + const expected = { name: 'ts569-durable-' + owner, owner, id: 'a'.repeat(64) } + const actual = { + Name: '/' + expected.name, + Id: expected.id, + Config: { + Image: docker.image, + Labels: { 'network-ops.fixture': 'ts-stack-544-durable', 'network-ops.fixture-owner': expected.owner } + } + } + expect(() => docker.validateContainer(actual, expected)).not.toThrow() + // A lost create reply may omit the ID, but never the complete ownership claim. + expect(() => docker.validateContainer(actual, { name: expected.name, owner: expected.owner })).not.toThrow() + for (const changed of [ + { ...actual, Name: '/another' }, + { ...actual, Id: 'b'.repeat(64) }, + { ...actual, Config: { ...actual.Config, Image: 'mysql:latest' } }, + { ...actual, Config: { ...actual.Config, Labels: { ...actual.Config.Labels, 'network-ops.fixture': 'another' } } }, + { + ...actual, + Config: { ...actual.Config, Labels: { ...actual.Config.Labels, 'network-ops.fixture-owner': 'another' } } + } + ]) + expect(() => docker.validateContainer(changed, expected)).toThrow() + expect(() => + docker.validateContainer({ ...actual, Id: '' }, { name: expected.name, owner: expected.owner }) + ).toThrow() +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveGuardChild.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveGuardChild.cjs new file mode 100644 index 000000000..909ab7cfc --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveGuardChild.cjs @@ -0,0 +1,22 @@ +// Owned synthetic process fixture; invoked only by snapshotArchiveGuardCrash.cjs. +const { knex } = require('knex') +const { readGuardedSnapshotArchive } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveGuard.js') +if (typeof process.send !== 'function') throw new Error('Owned fixture parent required') +process.once('message', ({ config, owner }) => { + const control = knex(config) + const source = knex(config) + void readGuardedSnapshotArchive(control, source, owner, async trx => { + const row = await trx('snapshot_guard_fixture').first() + process.send({ ready: true, value: row.value }) + await new Promise(resolve => process.once('message', resolve)) + }) + .catch(error => { + process.send({ ready: false, error: error.name }) + process.exitCode = 1 + }) + .finally(async () => { + await source.destroy() + await control.destroy() + process.disconnect() + }) +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveGuardCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveGuardCrash.cjs new file mode 100644 index 000000000..f0dc44bbd --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveGuardCrash.cjs @@ -0,0 +1,113 @@ +// Native process-loss qualification, using only caller-owned synthetic databases. +const assert = require('node:assert/strict') +const { spawn } = require('node:child_process') +const { mkdtemp, rm } = require('node:fs/promises') +const { tmpdir } = require('node:os') +const { join } = require('node:path') +const { knex } = require('knex') +const { addSnapshotArchiveTables } = require('../../out/src/storage/schema/snapshotArchiveMigration.js') +const { addSnapshotArchiveRequestTable } = require('../../out/src/storage/schema/snapshotArchiveRequestMigration.js') +const { addSnapshotArchiveOwnerTable } = require('../../out/src/storage/schema/snapshotArchiveOwnerMigration.js') +const { addSnapshotArchiveGuardTable } = require('../../out/src/storage/schema/snapshotArchiveGuardMigration.js') +const { + KnexSnapshotArchiveRequestStore +} = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.js') +const { recoverSnapshotArchiveGuards } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveGuard.js') + +async function qualifyGuardProcessLoss(control, config) { + const requests = new KnexSnapshotArchiveRequestStore(control, true, true) + const identity = '02' + 'ee'.repeat(32) + const offered = await requests.offer(identity, { lifetimeMs: 300000, maxBytes: 32768 }) + const { owner } = await requests.claimReader(identity, offered.request) + await control.schema.createTable('snapshot_guard_fixture', table => { + table.integer('id').primary() + table.string('value') + }) + await control('snapshot_guard_fixture').insert({ id: 1, value: 'before' }) + const child = spawn(process.execPath, [join(__dirname, 'snapshotArchiveGuardChild.cjs')], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'] + }) + let stderr = '' + child.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-65536) + }) + const exited = new Promise((resolve, reject) => { + child.once('exit', (code, signal) => resolve({ code, signal })) + child.once('error', reject) + }) + try { + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error('Owned guard child readiness deadline')), 15000) + child.once('message', message => { + clearTimeout(timer) + try { + assert.deepEqual(message, { ready: true, value: 'before' }) + resolve() + } catch (error) { + reject(error) + } + }) + void exited.then( + () => { + clearTimeout(timer) + reject(new Error(stderr || 'Owned guard child exited before readiness')) + }, + error => { + clearTimeout(timer) + reject(error) + } + ) + // Ephemeral fixture connection details travel through private IPC, never command arguments or output. + child.send({ config, owner }) + }) + await requests.markReaderCancellation(identity, offered.request) + await recoverSnapshotArchiveGuards(control, config) + await requests.reap() + assert.equal((await control('snapshot_archive_owners')).length, 1) + assert.equal(Number((await control('snapshot_archive_capacity').first()).archives), 1) + await control('snapshot_guard_fixture').update({ value: 'survives owner loss' }) + child.kill('SIGTERM') + const result = await exited + assert.equal(result.signal, 'SIGTERM') + await recoverSnapshotArchiveGuards(control, config) + await requests.reap() + assert.equal((await control('snapshot_archive_owners')).length, 0) + assert.equal(Number((await control('snapshot_archive_capacity').first()).archives), 0) + assert.equal((await control('snapshot_guard_fixture').first()).value, 'survives owner loss') + await assert.rejects(requests.claimReader(identity, offered.request), /unavailable/) + return { + actualSignal: result.signal, + busyBeforeLoss: true, + foregroundWrite: true, + exactOwnerRecovered: true, + staleRequestRefused: true + } + } finally { + if (child.exitCode === null && child.signalCode === null) child.kill('SIGTERM') + await exited + await control.schema.dropTable('snapshot_guard_fixture') + } +} + +async function qualifySQLiteGuardProcessLoss() { + const directory = await mkdtemp(join(tmpdir(), 'ts569-guard-process-')) + const config = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + } + const control = knex(config) + try { + await control.raw('PRAGMA journal_mode = WAL') + await addSnapshotArchiveTables(control) + await addSnapshotArchiveRequestTable(control) + await addSnapshotArchiveOwnerTable(control) + await addSnapshotArchiveGuardTable(control) + return await qualifyGuardProcessLoss(control, config) + } finally { + await control.destroy() + await rm(directory, { recursive: true, force: true }) + } +} +module.exports = { qualifyGuardProcessLoss, qualifySQLiteGuardProcessLoss } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveMysql.cjs new file mode 100644 index 000000000..9034e790e --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveMysql.cjs @@ -0,0 +1,640 @@ +const assert = require('node:assert/strict') +const { execFileSync } = require('node:child_process') +const { knex } = require('knex') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { SnapshotArchiveCleanupPendingError } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveOwner.js') +const { executable, context, validateContext, validateContainer } = require('./snapshotArchiveDocker.cjs') +const container = process.env.TS_STACK_SNAPSHOT_CONTAINER +const expectedId = process.env.TS_STACK_SNAPSHOT_CONTAINER_ID +const owner = process.env.TS_STACK_SNAPSHOT_CONTAINER_OWNER +const secret = process.env.TS_STACK_SNAPSHOT_MYSQL_SECRET +if (!container || !expectedId || !owner || !secret) throw new Error('Use the bounded fixture launcher') +const docker = (...args) => + execFileSync(executable, ['--context', context, ...args], { encoding: 'utf8', timeout: 15000 }) +validateContext(JSON.parse(docker('context', 'inspect', context))) +const actual = JSON.parse(docker('inspect', container))[0] +validateContainer(actual, { name: container, owner, id: expectedId }) +const port = Number(docker('port', expectedId, '3306/tcp').trim().split(':').at(-1)) +const connection = { + host: '127.0.0.1', + port, + user: 'root', + password: secret, + database: 'ts569_snapshot', + timezone: 'Z' +} + +const { + KnexSnapshotArchiveStore, + snapshotArchiveTables, + snapshotArchiveLimits +} = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveStore.js') +const { addSnapshotArchiveTables } = require('../../out/src/storage/schema/snapshotArchiveMigration.js') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { exportBRC38 } = require('../../out/src/storage/portable/index.js') +const { captureKnexSnapshotArchive } = require('../../out/src/storage/snapshot/archive/captureKnexSnapshotArchive.js') +const { decodeSyncTransfer } = require('../../out/src/storage/remoting/SyncTransfer.js') +const { KnexSnapshotArchiveService } = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveService.js') +const { + KnexSnapshotArchiveRequestStore +} = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveRequestStore.js') +const { snapshotArchiveRequestId } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveRequest.js') +const { snapshotArchiveDatabaseNow } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveSql.js') +const { KnexSnapshotArchiveRpc } = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveRpc.js') +const { SnapshotArchiveTransport } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveTransport.js') +const { openRemoteSnapshot } = require('../../out/src/storage/snapshot/archive/openRemoteSnapshot.js') +const { + SnapshotArchiveTransportFailure +} = require('../../out/src/storage/snapshot/archive/SnapshotArchiveTransportFailure.js') +const { + verifySnapshotArchiveDirectory, + verifySnapshotArchivePage +} = require('../../out/src/storage/snapshot/archive/SnapshotArchiveDirectory.js') +const open = () => knex({ client: 'mysql2', connection, pool: { min: 1, max: 1 } }) +const database = open(), + replica = open(), + third = open() +const identity = '02' + '11'.repeat(32), + other = '03' + '22'.repeat(32) +const date = new Date('2026-01-01T00:00:00.000Z') +const binding = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceSchema: 'synthetic-v1', + sourceStorage: { + created_at: date, + updated_at: date, + storageIdentityKey: 'source', + storageName: 'source', + chain: 'test', + dbtype: 'MySQL', + maxOutputScript: 1024 + }, + user: { created_at: date, updated_at: date, userId: 7, identityKey: identity, activeStorage: 'source' } +} + +async function captureFixture() { + const writer = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: open() }) + const reader = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: open() }) + const originalAppend = KnexSnapshotArchiveStore.prototype.append + try { + await writer.migrate('native capture source', 'native-source') + await writer.makeAvailable() + await reader.makeAvailable() + const { user } = await writer.findOrInsertUser(identity) + const { user: foreign } = await writer.findOrInsertUser(other) + const labels = Array.from({ length: 140 }, (_, index) => ({ + userId: user.userId, + label: `original-${index}`, + isDeleted: index % 7 === 0, + created_at: date, + updated_at: date + })) + await writer.knex('tx_labels').insert(labels) + await writer.findOrInsertTxLabel(foreign.userId, 'foreign label') + const tx = { + created_at: date, + updated_at: date, + status: 'completed', + isOutgoing: false, + satoshis: 1, + description: 'synthetic fixture' + } + const proof = await writer.insertProvenTx({ + created_at: date, + updated_at: date, + provenTxId: 0, + txid: 'a'.repeat(64), + height: 1, + index: 0, + merklePath: [4, 5, 255], + rawTx: [1, 2, 255], + blockHash: 'b'.repeat(64), + merkleRoot: 'c'.repeat(64) + }) + await writer.insertTransaction({ + ...tx, + transactionId: 0, + userId: user.userId, + provenTxId: proof, + txid: 'a'.repeat(64), + reference: 'owned' + }) + const foreignTransaction = await writer.insertTransaction({ + ...tx, + transactionId: 0, + userId: foreign.userId, + reference: 'foreign' + }) + await writer.knex('users').where({ userId: user.userId }).update({ activeStorage: 'historical selection' }) + const portable = await exportBRC38(reader, identity, { requireSnapshot: true }) + assert.equal(portable.user.activeStorage, 'historical selection') + assert.equal(portable.tables.provenTxs.length, 1) + assert.equal(portable.tables.provenTxs[0].rawTx, 'AQL/') + await reader.readSnapshot(async trx => { + assert.equal((await reader.findProvenTxs({ partial: { provenTxId: proof }, trx })).length, 1) + assert.deepEqual( + await reader.findSyncStates({ partial: { userId: user.userId, storageIdentityKey: 'native-source' }, trx }), + [] + ) + }) + let changedDuringCapture = false + KnexSnapshotArchiveStore.prototype.append = async function (owner, page, authorize) { + await originalAppend.call(this, owner, page, authorize) + if (page.sequence === 0) { + await writer + .knex('tx_labels') + .where({ userId: user.userId, label: 'original-0' }) + .update({ label: 'replacement' }) + await writer.knex('users').where({ userId: user.userId }).update({ activeStorage: 'replacement selection' }) + changedDuringCapture = true + } + } + const manifest = await captureKnexSnapshotArchive(reader, writer.knex, identity, 'test') + KnexSnapshotArchiveStore.prototype.append = originalAppend + assert.equal(changedDuringCapture, true) + assert.equal(manifest.pages, 14) + assert.equal(manifest.binding.sourceStorage.storageIdentityKey, 'native-source') + assert.equal(manifest.binding.user.activeStorage, 'historical selection') + assert.equal(manifest.binding.sourceSchema, '2026-10-02-001 repair snapshot SQLite conflict maintenance') + const store = new KnexSnapshotArchiveStore(writer.knex) + const first = decodeSyncTransfer((await store.read(identity, manifest.archiveId, 8)).bytes) + const second = decodeSyncTransfer((await store.read(identity, manifest.archiveId, 9)).bytes) + assert.equal(first.table, 'txLabels') + assert.equal(first.rows.length, 128) + assert.equal(second.rows.length, 12) + assert.deepEqual( + [...first.rows, ...second.rows].map(row => row.label), + labels.map(row => row.label) + ) + assert.equal(first.rows[0].created_at, date.toISOString()) + assert.equal(first.rows[0].isDeleted, true) + const proofPage = decodeSyncTransfer((await store.read(identity, manifest.archiveId, 0)).bytes) + assert.deepEqual(proofPage.rows[0].rawTx, new Uint8Array([1, 2, 255])) + await store.close(identity, manifest.archiveId) + const requestLifecycle = await requestFixture(writer, reader) + const remoteReader = await readerFixture(writer, reader) + const ownerDrain = await ownerDrainFixture(writer, reader) + const { qualifyGuardProcessLoss } = require('./snapshotArchiveGuardCrash.cjs') + const ownerProcessLoss = await qualifyGuardProcessLoss(writer.knex, { + client: 'mysql2', + connection, + pool: { min: 0, max: 1 } + }) + await writer.insertCommission({ + created_at: date, + updated_at: date, + commissionId: 0, + userId: user.userId, + transactionId: foreignTransaction, + satoshis: 1, + keyOffset: 'synthetic', + isRedeemed: false, + lockingScript: [1] + }) + await assert.rejects(captureKnexSnapshotArchive(reader, writer.knex, identity, 'test'), /relation/) + assert.equal((await writer.knex('snapshot_archive_capacity').first()).archives, 0) + return { + tables: 13, + pages: manifest.pages, + labels: 140, + pinnedConcurrentWrites: true, + originalPrimary: true, + originalSchema: true, + packedBinary: true, + portableProofAndCheckpointReadOnly: true, + crossProfileClosureRejected: true, + requestLifecycle, + remoteReader, + ownerDrain, + ownerProcessLoss + } + } finally { + KnexSnapshotArchiveStore.prototype.append = originalAppend + await reader.destroy() + await writer.destroy() + } +} +function boundary() { + let resolve + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} + +async function ownerDrainFixture(writer, reader) { + const controller = new KnexSnapshotArchiveService(writer) + const replacement = new KnexSnapshotArchiveService(reader) + const originalOpen = writer.openSnapshotArchiveSource.bind(writer) + const reading = boundary() + const allowRead = boundary() + const destroying = boundary() + const allowDestroy = boundary() + const fields = { + version: 1, + nonce: 'd'.repeat(64), + notAfter: (await snapshotArchiveDatabaseNow(writer.knex)) + 300000, + maxBytes: 1048576 + } + const input = { ...fields, requestId: snapshotArchiveRequestId(fields) } + writer.openSnapshotArchiveSource = async (...args) => { + const source = await originalOpen(...args) + assert.ok(source) + const pool = writer.snapshotSyncSource + assert.ok(pool) + const destroy = pool.knex.client.destroyRawConnection.bind(pool.knex.client) + pool.knex.client.destroyRawConnection = async connection => { + destroying.resolve() + await allowDestroy.promise + await destroy(connection) + } + return { + ...source, + readPage: async (...pageArgs) => { + reading.resolve() + await allowRead.promise + return await source.readPage(...pageArgs) + } + } + } + const capture = controller.create(identity, input) + void capture.catch(() => undefined) + try { + await reading.promise + await assert.rejects(replacement.cancelRequest(identity, input), SnapshotArchiveCleanupPendingError) + assert.equal((await replacement.status(identity, input.requestId)).state, 'closed') + const requests = new KnexSnapshotArchiveRequestStore(reader.knex, true) + await requests.reap() + await new KnexSnapshotArchiveStore(reader.knex).reap() + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).reservedBytes), fields.maxBytes) + assert.equal((await writer.knex('snapshot_archive_owners')).length, 1) + allowRead.resolve() + await destroying.promise + assert.equal((await writer.knex('snapshot_archive_pages')).length, 0) + assert.equal((await writer.knex('snapshot_archive_owners')).length, 1) + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).reservedBytes), fields.maxBytes) + await writer.knex('tx_labels').where({ label: 'after-reader-pin' }).update({ label: 'during-owner-drain' }) + assert.equal((await writer.knex('tx_labels').where({ label: 'during-owner-drain' })).length, 1) + allowDestroy.resolve() + await assert.rejects(capture, /unavailable/) + assert.equal(writer.snapshotSyncSource, undefined) + assert.equal((await writer.knex('snapshot_archive_owners')).length, 0) + assert.equal((await writer.knex('snapshot_archives')).length, 0) + assert.equal((await writer.knex('snapshot_archive_pages')).length, 0) + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).reservedBytes), 0) + await replacement.cancelRequest(identity, input) + return { + crossControllerCancellation: true, + nextAppendFenced: true, + reapingRetainsQuota: true, + foregroundWriteDuringPoolDrain: true, + exactPhysicalCleanupAcknowledgement: true + } + } finally { + allowRead.resolve() + allowDestroy.resolve() + await capture.catch(() => undefined) + writer.openSnapshotArchiveSource = originalOpen + await Promise.all([controller.close(), replacement.close()]) + } +} + +async function readerFixture(writer, reader) { + const owner = new KnexSnapshotArchiveRpc(writer) + const replacement = new KnexSnapshotArchiveRpc(reader) + const originalOpen = writer.openSnapshotArchiveSource.bind(writer) + const retained = Number((await writer.knex('snapshot_archive_requests').count({ count: '*' }).first()).count) + let captures = 0 + let receiver = owner + let admissionLost = false + const admissions = [] + writer.openSnapshotArchiveSource = async (...args) => { + captures++ + return await originalOpen(...args) + } + const transport = new SnapshotArchiveTransport( + async (method, params) => { + const value = await receiver.dispatch(method, params, identity) + if (method === 'admitSnapshotArchive') { + admissions.push(params) + if (!admissionLost) { + admissionLost = true + throw new SnapshotArchiveTransportFailure('synthetic lost admission acknowledgement') + } + } + return value + }, + identity, + 'native-source', + 'test', + true + ) + try { + await runInSeries([0, 1, 2, 3, 4], async iteration => { + receiver = owner + const view = await openRemoteSnapshot(transport) + assert.ok(view) + try { + assert.equal(view.sourceStorage.dbtype, 'MySQL') + assert.equal(view.user.identityKey, identity) + assert.ok(view.user.created_at instanceof Date) + assert.equal(writer.snapshotSyncSource, undefined) + receiver = replacement + if (iteration === 0) + await writer.knex('tx_labels').where({ label: 'after-service-pin' }).update({ label: 'after-reader-pin' }) + const labels = [] + let cursor + let done = false + function* pendingPages() { + while (!done) yield cursor + } + await runInSeries(pendingPages(), async next => { + const page = await view.readPage('txLabels', next, { maxRows: 17 }) + assert.ok(page.rows.length <= 17) + assert.equal(page.cursor.archivePosition.version, 1) + assert.equal(page.cursor.archivePosition.archiveId.length, 64) + labels.push(...page.rows) + cursor = page.cursor + done = page.done + }) + assert.equal(labels.length, 140) + assert.equal(labels[0].label, iteration === 0 ? 'after-service-pin' : 'after-reader-pin') + assert.equal(new Set(labels.map(row => row.txLabelId)).size, 140) + assert.ok(labels.every(row => row.userId === view.user.userId && row.created_at instanceof Date)) + assert.ok(labels.every(row => typeof row.isDeleted === 'boolean')) + await runInSeries(snapshotArchiveTables, async table => { + const page = await view.readPage(table, undefined, { maxRows: 1000 }) + assert.equal(page.done, true) + if (table === 'provenTxs') assert.deepEqual(page.rows[0].rawTx, new Uint8Array([1, 2, 255])) + }) + } finally { + await view.close() + } + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).reservedBytes), 0) + assert.equal((await writer.knex('snapshot_archive_requests')).length, retained) + assert.equal((await writer.knex('snapshot_archive_pages')).length, 0) + }) + assert.equal(captures, 5) + assert.equal(admissions.length, 6) + assert.deepEqual(admissions[0], admissions[1]) + const issued = await transport.readerOffer({ lifetimeMs: 300000, maxBytes: 1048576 }) + assert.equal(issued.outcome, 'offered') + await transport.cancelRequest(issued.request) + await assert.rejects(transport.admit(issued.request), /unavailable/) + assert.equal(captures, 5) + assert.equal((await writer.knex('snapshot_archive_requests')).length, retained) + return { + transport: 'profile-bound dispatcher over independent MySQL connections; HTTP qualified separately', + tables: 13, + captures, + maxPageRows: 17, + pinnedConcurrentWrite: true, + packedBytesAndDates: true, + replacementReader: true, + exactLostAdmissionRetry: true, + readerReceiptsCollected: true, + cancellationBeforeAdmission: true + } + } finally { + writer.openSnapshotArchiveSource = originalOpen + await Promise.all([owner.close(), replacement.close()]) + } +} +async function requestFixture(writer, reader) { + const controller = new KnexSnapshotArchiveService(writer) + const replacement = new KnexSnapshotArchiveService(reader) + const fields = { + version: 1, + nonce: 'b'.repeat(64), + notAfter: (await snapshotArchiveDatabaseNow(writer.knex)) + 300000, + maxBytes: 1048576 + } + const input = { ...fields, requestId: snapshotArchiveRequestId(fields) } + const originalOpen = writer.openSnapshotArchiveSource.bind(writer) + const originalAppend = KnexSnapshotArchiveStore.prototype.append + let claimedBeforePool = false + let recoveredWhileCapturing = false + writer.openSnapshotArchiveSource = async (...args) => { + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).reservedBytes), fields.maxBytes) + claimedBeforePool = true + return await originalOpen(...args) + } + KnexSnapshotArchiveStore.prototype.append = async function (owner, page, authorize) { + await originalAppend.call(this, owner, page, authorize) + if (page.sequence === 0) { + const receipt = await replacement.create(identity, input) + assert.equal(receipt.state, 'building') + assert.equal(receipt.requestId, input.requestId) + recoveredWhileCapturing = true + await writer.knex('tx_labels').where({ label: 'replacement' }).update({ label: 'after-service-pin' }) + } + } + try { + const pending = controller.create(identity, input) + assert.equal(controller.create(identity, { ...input }), pending) + const ready = await pending + assert.equal(ready.state, 'ready') + assert.equal(ready.expiresAt, fields.notAfter) + assert.equal(claimedBeforePool, true) + assert.equal(recoveredWhileCapturing, true) + const directory = await replacement.directory(identity, ready.archiveId) + const verified = verifySnapshotArchiveDirectory(directory, { + identityKey: identity, + chain: 'test', + sourceStorageIdentityKey: 'native-source', + digest: ready.digest + }) + assert.equal(verified.manifest.binding.sourceSchema, '2026-10-02-001 repair snapshot SQLite conflict maintenance') + const page = await replacement.read(identity, ready.archiveId, 8) + const decoded = decodeSyncTransfer(verifySnapshotArchivePage(page, verified.receipts[8])) + assert.equal(decoded.rows[0].label, 'replacement') + assert.equal(writer.snapshotSyncSource, undefined) + await controller.close() + assert.deepEqual(await replacement.create(identity, input), ready) + await assert.rejects(replacement.status(other, input.requestId), /unavailable/) + const requests = new KnexSnapshotArchiveRequestStore(writer.knex) + await Promise.all([replacement.cancel(identity, input.requestId), requests.close(identity, input.requestId)]) + assert.equal((await replacement.create(identity, input)).state, 'closed') + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).reservedBytes), 0) + const nextFields = { ...fields, nonce: 'c'.repeat(64) } + const next = { ...nextFields, requestId: snapshotArchiveRequestId(nextFields) } + const admitted = await requests.claim(identity, next) + await writer.knex.raw( + "CREATE TRIGGER synthetic_request_assignment_failure BEFORE UPDATE ON snapshot_archive_requests FOR EACH ROW BEGIN IF NEW.archiveId IS NOT NULL THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='synthetic request assignment failure'; END IF; END" + ) + try { + await assert.rejects( + requests.begin(admitted.owner, verified.manifest.binding), + /synthetic request assignment failure/ + ) + } finally { + await writer.knex.raw('DROP TRIGGER synthetic_request_assignment_failure') + } + assert.equal( + (await writer.knex('snapshot_archive_requests').where({ requestId: next.requestId }).first()).state, + 'claimed' + ) + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).reservedBytes), fields.maxBytes) + assert.equal((await writer.knex('snapshot_archives')).length, 0) + await requests.close(identity, next.requestId) + assert.equal(Number((await writer.knex('snapshot_archive_capacity').first()).archives), 0) + return { + claimedBeforePool, + recoveredWhileCapturing, + fixedDeadline: true, + verifiedDirectoryAndPage: true, + foregroundWrite: true, + replacementReadyReceipt: true, + concurrentClose: true, + atomicAssignmentRollback: true + } + } finally { + writer.openSnapshotArchiveSource = originalOpen + KnexSnapshotArchiveStore.prototype.append = originalAppend + await Promise.all([controller.close(), replacement.close()]) + } +} +const { mysqlFixtureGroups } = require('./snapshotMysqlFixtureGroups.cjs') +const indexFixtures = { + profile: async () => { + const { qualifyMysqlProfileIndexProcessLoss } = require('./snapshotProfileIndexCrash.cjs') + const profileIndexProcessLoss = await qualifyMysqlProfileIndexProcessLoss(database, connection) + const { qualifyMysqlProfileIndexLocks } = require('./snapshotProfileIndexMysql.cjs') + const profileIndexLocks = await qualifyMysqlProfileIndexLocks(database, connection) + return { profileIndexProcessLoss, profileIndexLocks } + }, + relation: async () => { + const { qualifyMysqlRelationIndexProcessLoss } = require('./snapshotRelationIndexCrash.cjs') + const relationIndexProcessLoss = await qualifyMysqlRelationIndexProcessLoss(database, connection) + const { qualifyMysqlRelationIndexLocks } = require('./snapshotRelationIndexMysql.cjs') + const relationIndexLocks = await qualifyMysqlRelationIndexLocks(database, connection) + const { qualifyMysqlRelationIndexSeeks } = require('./snapshotRelationIndexSeeks.cjs') + const relationIndexSeeks = await qualifyMysqlRelationIndexSeeks(database, connection) + return { relationIndexProcessLoss, relationIndexLocks, relationIndexSeeks } + }, + certificate: async () => { + const { qualifyMysqlCertificateIndexProcessLoss } = require('./snapshotCertificateIndexCrash.cjs') + const certificateIndexProcessLoss = await qualifyMysqlCertificateIndexProcessLoss(database, connection) + const { qualifyMysqlCertificateIndexLocks } = require('./snapshotCertificateIndexMysql.cjs') + const certificateIndexLocks = await qualifyMysqlCertificateIndexLocks(database, connection) + const { qualifyMysqlCertificateIndexSchedules } = require('./snapshotCertificateIndexMysqlSchedules.cjs') + const certificateIndexSchedules = await qualifyMysqlCertificateIndexSchedules(database, connection) + const { qualifyMysqlCertificateIndexSeeks } = require('./snapshotCertificateIndexSeeks.cjs') + const certificateIndexSeeks = await qualifyMysqlCertificateIndexSeeks(database, connection) + return { certificateIndexProcessLoss, certificateIndexLocks, certificateIndexSchedules, certificateIndexSeeks } + }, + 'global-crash': async () => { + const { qualifyMysqlGlobalIndexProcessLoss } = require('./snapshotGlobalIndexCrash.cjs') + return { globalIndexProcessLoss: await qualifyMysqlGlobalIndexProcessLoss(database, connection) } + }, + 'global-locks': async () => { + const { qualifyMysqlGlobalIndexLocks } = require('./snapshotGlobalIndexMysql.cjs') + return { globalIndexLocks: await qualifyMysqlGlobalIndexLocks(database, connection) } + }, + 'global-schedules': async () => { + const { qualifyMysqlGlobalIndexSchedules } = require('./snapshotGlobalIndexMysqlSchedules.cjs') + return { globalIndexSchedules: await qualifyMysqlGlobalIndexSchedules(database, connection) } + }, + 'global-seeks': async () => { + const { qualifyMysqlGlobalIndexSeeks } = require('./snapshotGlobalIndexSeeks.cjs') + return { globalIndexSeeks: await qualifyMysqlGlobalIndexSeeks(database, connection) } + }, + 'global-integration': async () => { + const { qualifyMysqlGlobalIndexIntegration } = require('./snapshotGlobalIndexIntegration.cjs') + return { globalIndexIntegration: await qualifyMysqlGlobalIndexIntegration(database, connection) } + } +} +async function runIndexFixtures(group) { + const results = {} + const groups = group === 'all' ? mysqlFixtureGroups.slice(1) : [group] + await runInSeries(groups, async name => { + Object.assign(results, await indexFixtures[name]()) + }) + return results +} +async function main() { + const group = process.argv[2] ?? 'all' + try { + assert.ok(group === 'all' || mysqlFixtureGroups.includes(group), 'Unknown native fixture group') + const version = (await database.raw('SELECT VERSION() AS version'))[0][0].version + if (group !== 'archive' && group !== 'all') { + console.log(JSON.stringify({ version, group, ...(await runIndexFixtures(group)) })) + return + } + await addSnapshotArchiveTables(database) + const store = new KnexSnapshotArchiveStore(database), + peer = new KnexSnapshotArchiveStore(replica), + final = new KnexSnapshotArchiveStore(third) + const writer = await store.begin(binding) + await assert.rejects(peer.inspect(identity, writer.archiveId), /unavailable/) + await assert.rejects(peer.begin(binding), /occupied/) + const payload = new Uint8Array(snapshotArchiveLimits.pageBytes).fill(42) + await runInSeries(snapshotArchiveTables.entries(), ([sequence, table]) => + store.append(writer, { sequence, table, rows: 1, done: true, bytes: payload }) + ) + const manifest = await store.seal(writer) + assert.deepEqual(manifest.binding, binding) + assert.equal(manifest.pages, 13) + assert.equal(manifest.rows, 13) + assert.deepEqual(await peer.inspect(identity, writer.archiveId), manifest) + assert.deepEqual((await peer.read(identity, writer.archiveId, 0)).bytes, payload) + await assert.rejects(peer.read(other, writer.archiveId, 0), /unavailable/) + await addSnapshotArchiveTables(database) + assert.equal(Number((await database('snapshot_archive_capacity').first()).archives), 1) + await Promise.all([peer.close(identity, writer.archiveId), final.close(identity, writer.archiveId)]) + assert.equal(Number((await database('snapshot_archive_pages').count({ count: '*' }))[0].count), 0) + assert.equal(Number((await database('snapshot_archive_capacity').first()).reservedBytes), 0) + // SQL failure after page insertion must roll back both the page and cursor. + const failing = await store.begin(binding) + await database.raw( + "CREATE TRIGGER synthetic_archive_failure BEFORE UPDATE ON snapshot_archives FOR EACH ROW BEGIN IF NEW.nextSequence > OLD.nextSequence THEN SIGNAL SQLSTATE '45000' SET MESSAGE_TEXT='synthetic append failure'; END IF; END" + ) + await assert.rejects( + store.append(failing, { sequence: 0, table: 'provenTxs', rows: 0, done: true, bytes: new Uint8Array([91, 93]) }), + /synthetic append failure/ + ) + await database.raw('DROP TRIGGER synthetic_archive_failure') + assert.equal(Number((await database('snapshot_archive_pages').count({ count: '*' }))[0].count), 0) + assert.equal((await database('snapshot_archives').where({ archiveId: failing.archiveId }).first()).nextSequence, 0) + await database('snapshot_archives').where({ archiveId: failing.archiveId }).update({ expiresAt: 0 }) + await peer.reap() + assert.equal(Number((await database('snapshot_archive_capacity').first()).archives), 0) + // Independent servers racing for one profile can admit exactly one capture. + const contested = await Promise.allSettled([store.begin(binding), peer.begin(binding), final.begin(binding)]) + assert.equal(contested.filter(x => x.status === 'fulfilled').length, 1) + const admitted = contested.find(x => x.status === 'fulfilled').value + await peer.close(identity, admitted.archiveId) + await database.schema.dropTable('snapshot_archive_pages') + await addSnapshotArchiveTables(database) + assert.equal(await database.schema.hasTable('snapshot_archive_pages'), true) + assert.equal(Number((await database('snapshot_archive_capacity').first()).archives), 0) + const capture = await captureFixture() + const indexes = group === 'all' ? await runIndexFixtures(group) : {} + console.log( + JSON.stringify({ + version, + source: 'built checkout; internal staging only', + crossReplicaImmutableReads: true, + exactMaximumPageBytes: payload.length, + profileOwnership: true, + concurrentCloseExactlyOnce: true, + appendRollback: true, + expiredPartialUnreadable: true, + profileReservationRace: true, + idempotentPartialDdl: true, + ...indexes, + capture + }) + ) + } finally { + await third.destroy() + await replica.destroy() + await database.destroy() + } +} +main().catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveMysqlLauncher.test.ts b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveMysqlLauncher.test.ts new file mode 100644 index 000000000..35e91ab3f --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotArchiveMysqlLauncher.test.ts @@ -0,0 +1,234 @@ +import assert from 'node:assert/strict' +import { readFileSync } from 'node:fs' +import { join } from 'node:path' +import { runInNewContext } from 'node:vm' + +const source = readFileSync(join(__dirname, 'runSnapshotArchiveMysql.cjs'), 'utf8') +const dockerSource = readFileSync(join(__dirname, 'snapshotArchiveDocker.cjs'), 'utf8') +const groupsSource = readFileSync(join(__dirname, 'snapshotMysqlFixtureGroups.cjs'), 'utf8') +const owner = '00000000-0000-4000-8000-000000000001' +const id = 'a'.repeat(64) +type Failure = + 'none' | 'image' | 'create-reply' | 'child' | 'late-child' | 'cleanup-identity' | 'cancel' | 'late-cancel' +interface Call { + executable: string + args: string[] + options: Record +} +function fixture(failure: Failure) { + const helper = { exports: {} } + runInNewContext(dockerSource, { + module: helper, + process: { platform: 'darwin', env: {} }, + require: (name: string) => (name === 'node:assert/strict' ? assert : { existsSync: () => true }) + }) + const groups = { exports: {} } + runInNewContext(groupsSource, { module: groups }) + const docker = helper.exports as { image: string } + const metadata = { + Id: id, + Name: '/ts569-durable-' + owner, + Config: { + Image: docker.image, + Labels: { 'network-ops.fixture': 'ts-stack-544-durable', 'network-ops.fixture-owner': owner } + } + } + const calls: Call[] = [] + const signals = new Map void>() + const error = new Error('synthetic ' + failure) + let exists = false + let childFinished = false + const perform = (executable: string, args: string[], options: Record) => { + calls.push({ executable, args, options }) + if (executable === '/synthetic/node') { + childFinished = true + if (failure === 'child' || (failure === 'late-child' && args[1] === 'global-schedules')) throw error + return 'synthetic native proof\n' + } + assert.deepEqual(Array.from(args.slice(0, 2)), ['--context', 'desktop-linux']) + switch (args[2]) { + case 'context': + return JSON.stringify([ + { Name: 'desktop-linux', Endpoints: { docker: { Host: 'unix:///synthetic/docker.sock' } } } + ]) + case 'image': + if (failure === 'image') throw error + return '[]' + case 'run': + exists = true + if (failure === 'create-reply') throw error + return id + case 'inspect': + return JSON.stringify([ + { ...metadata, Id: childFinished && failure === 'cleanup-identity' ? 'b'.repeat(64) : id } + ]) + case 'exec': + return 'mysqld is alive' + case 'ps': + return exists ? id : '' + case 'rm': + assert.equal(args[4], id) + exists = false + return id + default: + throw new Error('Unexpected Docker command') + } + } + const execFile = ( + executable: string, + args: string[], + options: Record, + callback: (error: unknown, stdout: string) => void + ) => { + if ( + executable === '/synthetic/node' && + (failure === 'cancel' || (failure === 'late-cancel' && args[1] === 'global-locks')) + ) { + calls.push({ executable, args, options }) + const signal = options.signal as AbortSignal + signal.addEventListener('abort', () => callback(error, ''), { once: true }) + queueMicrotask(() => signals.get('SIGTERM')?.()) + return + } + Promise.resolve() + .then(() => perform(executable, args, options)) + .then( + value => callback(null, value), + error => callback(error, '') + ) + } + const module = { exports: undefined as unknown } + const dependencies: Record = { + 'node:assert/strict': assert, + 'node:child_process': { execFile }, + 'node:crypto': { randomBytes: () => Buffer.alloc(32), randomUUID: () => owner }, + 'node:path': { join }, + './snapshotArchiveDocker.cjs': helper.exports, + './snapshotMysqlFixtureGroups.cjs': groups.exports, + '../../out/src/utility/runInSeries.js': { + runInSeries: async (items: Iterable, visit: (value: unknown) => Promise) => { + for (const item of items) await visit(item) + } + } + } + runInNewContext(source, { + module, + __dirname, + setTimeout, + AbortController, + process: { + env: {}, + execPath: '/synthetic/node', + stdout: { write: () => undefined }, + once: (name: string, listener: () => void) => signals.set(name, listener), + removeListener: (name: string) => signals.delete(name) + }, + require: (name: string) => { + assert.ok(Object.hasOwn(dependencies, name)) + return dependencies[name] + } + }) + return { run: module.exports as () => Promise, calls, error, exists: () => exists, signals } +} + +test('successful fixture retains bounds and proves exact-owner removal after its child finishes', async () => { + const f = fixture('none') + await expect(f.run()).resolves.toBeUndefined() + expect(f.exists()).toBe(false) + const creation = f.calls.find(call => call.args[2] === 'run')! + for (const value of [ + '--pull=never', + '127.0.0.1::3306', + '1g', + '2', + '256', + '/var/lib/mysql:rw,nosuid,nodev,size=512m', + 'network-ops.fixture-owner=' + owner + ]) { + expect(creation.args).toContain(value) + } + const children = f.calls.filter(call => call.executable === '/synthetic/node') + expect(children.map(child => child.args[1])).toEqual([ + 'archive', + 'profile', + 'relation', + 'certificate', + 'global-crash', + 'global-locks', + 'global-schedules', + 'global-seeks', + 'global-integration' + ]) + for (const child of children) { + expect(child.args[0]).toBe(join(__dirname, 'snapshotArchiveMysql.cjs')) + expect(child.options.timeout).toBe(60000) + expect(child.options.maxBuffer).toBe(1048576) + expect(child.options.env).toMatchObject({ + TS_STACK_SNAPSHOT_CONTAINER_OWNER: owner, + TS_STACK_SNAPSHOT_CONTAINER_ID: id + }) + } + expect( + f.calls.filter(call => call.executable !== '/synthetic/node').every(call => call.options.timeout === 15000) + ).toBe(true) + expect(f.calls.filter(call => call.args[2] === 'rm')).toHaveLength(1) +}) + +test.each(['create-reply', 'child', 'late-child'] as const)( + 'a %s failure removes only the claimed container and preserves the failure', + async failure => { + const f = fixture(failure) + await expect(f.run()).rejects.toBe(f.error) + expect(f.exists()).toBe(false) + expect(f.calls.filter(call => call.args[2] === 'rm')).toHaveLength(1) + const children = f.calls.filter(call => call.executable === '/synthetic/node') + const expected = { + 'create-reply': [], + child: ['archive'], + 'late-child': [ + 'archive', + 'profile', + 'relation', + 'certificate', + 'global-crash', + 'global-locks', + 'global-schedules' + ] + } + expect(children.map(child => child.args[1])).toEqual(expected[failure]) + } +) + +test('image inspection failure allocates no container and does not attempt removal', async () => { + const f = fixture('image') + await expect(f.run()).rejects.toBe(f.error) + expect(f.exists()).toBe(false) + expect(f.calls.some(call => ['run', 'rm'].includes(call.args[2]))).toBe(false) +}) + +test('an unexpected cleanup identity refuses removal and reports unproved cleanup', async () => { + const f = fixture('cleanup-identity') + await expect(f.run()).rejects.toThrow('cleanup is unproved') + expect(f.exists()).toBe(true) + expect(f.calls.some(call => call.args[2] === 'rm')).toBe(false) +}) + +test.each(['cancel', 'late-cancel'] as const)( + '%s awaits independently bounded cleanup and removes listeners', + async failure => { + const f = fixture(failure) + await expect(f.run()).rejects.toBe(f.error) + expect(f.exists()).toBe(false) + expect(f.signals.size).toBe(0) + const children = f.calls.filter(call => call.executable === '/synthetic/node') + expect(children.map(child => child.args[1])).toEqual( + failure === 'cancel' + ? ['archive'] + : ['archive', 'profile', 'relation', 'certificate', 'global-crash', 'global-locks'] + ) + expect((children.at(-1)!.options.signal as AbortSignal).aborted).toBe(true) + const removal = f.calls.find(call => call.args[2] === 'rm')! + expect(removal.options.signal).toBeUndefined() + expect(removal.options.timeout).toBe(15000) + } +) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexCrash.cjs new file mode 100644 index 000000000..155dd31b1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexCrash.cjs @@ -0,0 +1,257 @@ +const { migrateBeforeSqliteGeneration } = require('./snapshotHistoricalMigrations.cjs') +// Synthetic process-loss qualification, invoked by the existing native fixtures. +const assert = require('node:assert/strict') +const { spawn } = require('node:child_process') +const { randomUUID } = require('node:crypto') +const fs = require('node:fs/promises') +const { writeFileSync } = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { KnexMigrations } = require('../../out/src/storage/schema/KnexMigrations.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + readSnapshotCertificateIndexState, + SNAPSHOT_CERTIFICATE_INDEX_MIGRATION +} = require('../../out/src/storage/schema/snapshotCertificateIndexMigration.js') + +const phases = [ + 'after-key-table', + 'after-first-index', + 'partial-observers', + 'partial-producers', + 'before-cursor', + 'after-cursor', + 'after-commit' +] +const migrationName = 'synthetic certificate crash' +const migrationIdentity = 'synthetic-certificate-crash' +const { oracle } = require('./snapshotCertificateIndexMysqlSchedules.cjs') +const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } +const provider = options => + new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: knex(options) }) + +async function seed(options) { + const source = provider(options) + try { + if (options.client === 'better-sqlite3') await source.knex.raw('PRAGMA journal_mode = WAL') + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + await source.makeAvailable() + const migrationSource = new KnexMigrations('test', migrationName, migrationIdentity, 1024) + await source.knex.migrate.down({ + migrationSource, + name: SNAPSHOT_CERTIFICATE_INDEX_MIGRATION, + disableTransactions: false + }) + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await source.knex('certificates').insert({ + ...dates, + certificateId: 1, + userId: user.userId, + serialNumber: 'synthetic-certificate', + type: 'type', + certifier: '02' + '11'.repeat(32), + subject: '02' + '11'.repeat(32), + revocationOutpoint: 'a'.repeat(64) + '.0', + signature: 'synthetic', + isDeleted: false + }) + await runInSeries([0, 1, 2], async batch => { + await source.knex('certificate_fields').insert( + Array.from({ length: 200 }, (_, i) => ({ + ...dates, + userId: user.userId, + certificateId: 1, + fieldName: String(batch * 200 + i).padStart(6, '0'), + fieldValue: 'synthetic', + masterKey: 'synthetic' + })) + ) + }) + return { userId: user.userId, otherId: other.userId } + } finally { + await source.destroy() + } +} + +async function child(options, phase, marker) { + assert(phases.includes(phase)) + assert.equal(typeof process.send, 'function', 'Child requires its fixture parent') + const source = provider(options) + let cursorWritten = false + const cursor = query => + query.sql.startsWith('update `snapshot_certificate_index_progress`') && query.bindings.includes('000255') + const park = event => { + writeFileSync(marker, JSON.stringify({ phase, event }), { mode: 0o600 }) + process.kill(process.pid, 'SIGKILL') + } + source.knex.on('query', query => { + if (phase === 'before-cursor' && cursor(query)) park('query') + if (phase === 'after-commit' && cursorWritten && query.sql.toLowerCase().startsWith('begin')) + park('next-transaction') + }) + source.knex.on('query-response', (_result, query) => { + const sql = query.sql.toLowerCase() + if (phase === 'after-key-table' && sql.startsWith('create table `snapshot_certificate_field_keys`')) + park('query-response') + if ( + phase === 'after-first-index' && + (sql.startsWith('create index `snapshot_certificate_parent`') || + sql.startsWith('alter table `snapshot_certificate_field_keys` add index `snapshot_certificate_parent`')) + ) + park('query-response') + if (phase === 'partial-observers' && sql.startsWith('create trigger snapshot_certificate_field_delete ')) + park('query-response') + if (phase === 'partial-producers' && sql.startsWith('create trigger snapshot_certificate_field_insert ')) + park('query-response') + if (cursor(query)) { + cursorWritten = true + if (phase === 'after-cursor') park('query-response') + } + if (phase === 'after-commit' && cursorWritten && sql.startsWith('commit')) park('query-response') + }) + try { + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + throw new Error('Expected migration boundary was not reached') + } finally { + await source.destroy() + } +} + +async function terminateAt(options, phase) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-certificate-boundary-')) + const marker = path.join(directory, 'boundary.json') + const processHandle = spawn(process.execPath, [__filename, 'child'], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + let stderr = '' + processHandle.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-65536) + }) + const exited = new Promise((resolve, reject) => { + processHandle.once('exit', (code, signal) => resolve({ code, signal })) + processHandle.once('error', reject) + }) + const timer = setTimeout(() => processHandle.kill('SIGKILL'), 15000) + try { + processHandle.send({ options, phase, marker }) + const result = await exited + assert.equal(result.signal, 'SIGKILL', stderr) + const observed = JSON.parse(await fs.readFile(marker, 'utf8')) + assert.equal(observed.phase, phase) + return { phase, event: observed.event, signal: result.signal } + } finally { + clearTimeout(timer) + if (processHandle.exitCode === null && processHandle.signalCode === null) { + processHandle.kill('SIGKILL') + await exited + } + await fs.rm(directory, { recursive: true, force: true }) + } +} + +async function qualify(options, phase) { + const { userId, otherId } = await seed(options) + const outcome = await terminateAt(options, phase) + const source = provider(options) + const database = source.knex + try { + assert.equal(await readSnapshotCertificateIndexState(database), false) + if (['before-cursor', 'after-cursor', 'after-commit'].includes(phase)) { + const committed = phase === 'after-commit' + const progress = await database('snapshot_certificate_index_progress').where('snapshotTableId', 0).first() + assert.equal(progress.afterFieldName, committed ? '000255' : '') + assert.equal(progress.started, committed ? 1 : 0) + assert.equal(progress.afterCertificateId, committed ? 1 : 0) + assert.equal( + Number((await database('snapshot_certificate_field_keys').count({ count: '*' }).first()).count), + committed ? 256 : 0 + ) + } + await database('certificates').where('certificateId', 1).update({ userId: otherId }) + await database('certificate_fields').where('fieldName', '000004').delete() + await database('certificate_fields').insert({ + ...dates, + userId, + certificateId: 1, + fieldName: '000003a', + fieldValue: 'inserted-behind-cursor', + masterKey: 'synthetic' + }) + // A killed migrator leaves Knex's lock claimed. This is fixture-owned recovery; + // the verified child is gone and no other migrator can own this isolated store. + await database.migrate.forceFreeMigrationsLock() + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + assert.equal(await readSnapshotCertificateIndexState(database), true) + await oracle(database) + assert.equal( + Number( + ( + await database('knex_migrations') + .where('name', SNAPSHOT_CERTIFICATE_INDEX_MIGRATION) + .count({ count: '*' }) + .first() + ).count + ), + 1 + ) + return { ...outcome, journalPublishedAfterRecovery: true, independentProfileChangeAndLowIdInsert: true } + } finally { + await source.destroy() + } +} + +async function qualifySQLiteCertificateIndexProcessLoss() { + const results = [] + await runInSeries(phases, async phase => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-certificate-index-crash-')) + try { + results.push( + await qualify( + { + client: 'better-sqlite3', + connection: { filename: path.join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }, + phase + ) + ) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + return results +} + +async function qualifyMysqlCertificateIndexProcessLoss(control, connection) { + // The calling fixture has already verified its disposable, pinned container. + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const results = [] + await runInSeries(phases, async phase => { + const database = 'ts569_certificate_' + randomUUID().replaceAll('-', '') + await control.raw('CREATE DATABASE ??', [database]) + try { + results.push( + await qualify({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }, phase) + ) + } finally { + await control.raw('DROP DATABASE ??', [database]) + } + }) + return results +} + +if (process.argv[2] === 'child') { + assert.equal(typeof process.send, 'function', 'Child execution requires its fixture parent') + process.once('message', ({ options, phase, marker }) => { + child(options, phase, marker).catch(error => { + console.error(error) + process.exitCode = 1 + process.disconnect() + }) + }) +} +module.exports = { qualifySQLiteCertificateIndexProcessLoss, qualifyMysqlCertificateIndexProcessLoss } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexMysql.cjs new file mode 100644 index 000000000..26733ba0a --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexMysql.cjs @@ -0,0 +1,236 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +// Synthetic current-read and bootstrap-lock qualification on the native MySQL fixture. +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex } = require('knex') +const { oracle } = require('./snapshotCertificateIndexMysqlSchedules.cjs') +const helper = require('../../out/src/storage/schema/snapshotCertificateIndexMigration.js') +const install = helper.addSnapshotCertificateIndexes +const bootstrap = install +const settle = promise => + promise.then( + value => ({ ok: true, value }), + error => ({ ok: false, error }) + ) +const success = result => { + if (!result.ok) throw result.error +} +async function until(check) { + let observed + function* pending() { + for (let i = 0; i < 1000 && !observed; i++) yield i + } + await runInSeries(pending(), async () => { + observed = await check() + if (!observed) await new Promise(resolve => setTimeout(resolve, 5)) + }) + if (observed) return observed + throw new Error('Native lock observation exceeded five seconds') +} +async function fixture(admin, connection, isolation) { + const database = 'ts569_cert_locks_' + randomUUID().replaceAll('-', '') + await admin.raw('CREATE DATABASE ?? CHARACTER SET utf8mb4 COLLATE utf8mb4_0900_ai_ci', [database]) + const open = () => knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + const k = open(), + writer = open(), + observer = open(), + events = [] + let transaction + try { + await runInSeries([k, writer, observer], async client => { + await client.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation.toUpperCase()) + }) + await k.raw( + 'CREATE TABLE certificates(certificateId INT UNSIGNED PRIMARY KEY,userId INT UNSIGNED NOT NULL,isDeleted TINYINT NOT NULL DEFAULT 0) ENGINE=InnoDB' + ) + await k.raw( + 'CREATE TABLE certificate_fields(userId INT UNSIGNED NOT NULL,fieldName VARCHAR(100) NOT NULL,certificateId INT UNSIGNED NOT NULL,fieldValue VARCHAR(255) NOT NULL,UNIQUE(fieldName,certificateId)) ENGINE=InnoDB' + ) + await k('certificates').insert({ certificateId: 1, userId: 1 }) + await install(k) + const field = { userId: 1, fieldName: 'A', certificateId: 1, fieldValue: 'synthetic' } + const reset = async () => { + await k('certificate_fields').delete() + await k('certificates').where({ certificateId: 1 }).update({ userId: 1 }) + } + const verify = async () => { + await oracle(k) + assert.deepEqual( + await k('snapshot_certificate_field_keys') + .select({ + userId: 'snapshotUserId', + fieldName: 'snapshotFieldName', + certificateId: 'snapshotCertificateId', + membership: 'snapshotMembership' + }) + .orderBy('snapshotUserId'), + [ + { userId: 1, fieldName: 'A', certificateId: 1, membership: 1 }, + { userId: 2, fieldName: 'A', certificateId: 1, membership: 2 } + ] + ) + } + const id = async client => Number((await client.raw('SELECT CONNECTION_ID() AS id'))[0][0].id) + const kId = await id(k), + writerId = await id(writer) + const waiting = async requester => { + const [rows] = await observer.raw( + 'SELECT l.OBJECT_NAME AS tableName,l.LOCK_MODE AS lockMode,t.PROCESSLIST_ID AS requester FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE JOIN performance_schema.threads t ON t.THREAD_ID=l.THREAD_ID WHERE l.OBJECT_SCHEMA=DATABASE() AND t.PROCESSLIST_ID=?', + [requester] + ) + return rows.length ? rows : false + } + transaction = await writer.transaction() + await transaction('certificates').where({ certificateId: 1 }).update({ userId: 2 }) + let finished = false + const first = settle( + k('certificate_fields') + .insert(field) + .then(() => { + finished = true + }) + ) + try { + const locks = await until(() => waiting(kId)) + assert.equal(finished, false) + events.push({ case: 'field publication waits for current parent ownership', locks }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + success(await first) + await verify() + + await reset() + transaction = await writer.transaction() + await transaction('certificate_fields').insert(field) + finished = false + const second = settle( + k('certificates') + .where({ certificateId: 1 }) + .update({ userId: 2 }) + .then(() => { + finished = true + }) + ) + try { + const locks = await until(() => waiting(kId)) + assert.equal(finished, false) + assert.ok(locks.some(row => row.tableName === 'certificates')) + events.push({ case: 'parent update waits for field publication', locks }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + success(await second) + await verify() + + await reset() + transaction = await k.transaction() + try { + assert.equal((await transaction('certificates').first()).userId, 1) + await writer('certificates').where({ certificateId: 1 }).update({ userId: 2 }) + await transaction('certificate_fields').insert(field) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + await verify() + events.push({ case: 'field producer ignores earlier consistent parent snapshot' }) + + await reset() + transaction = await k.transaction() + try { + assert.equal((await transaction('snapshot_certificate_field_keys')).length, 0) + await writer('certificate_fields').insert(field) + await transaction('certificates').where({ certificateId: 1 }).update({ userId: 2 }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + await verify() + events.push({ case: 'parent producer uses current direct keys after an earlier consistent snapshot' }) + + const clearBootstrap = async () => { + await k('snapshot_certificate_field_keys').delete() + await k('snapshot_certificate_index_progress') + .where({ snapshotTableId: 0 }) + .update({ started: 0, afterFieldName: '', afterCertificateId: 0, complete: 0 }) + } + await clearBootstrap() + transaction = await writer.transaction() + await transaction('certificates').where({ certificateId: 1 }).update({ userId: 1 }) + await transaction('certificates').where({ certificateId: 1 }).update({ userId: 2 }) + const restoring = settle(bootstrap(k)) + try { + const locks = await until(() => waiting(kId)) + events.push({ case: 'bootstrap waits for current committed parent ownership', locks }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + success(await restoring) + await verify() + + await k('certificates').where({ certificateId: 1 }).update({ userId: 1 }) + await clearBootstrap() + let reached = false, + release + const gate = new Promise(resolve => { + release = resolve + }) + const prototype = Object.getPrototypeOf(k.client), + original = prototype.query, + hadOwn = Object.hasOwn(prototype, 'query') + prototype.query = async function (connection, query) { + const sql = typeof query === 'string' ? query : query.sql + if (!reached && sql.startsWith('update `snapshot_certificate_index_progress`')) { + reached = true + await gate + } + return await original.call(this, connection, query) + } + const pending = settle(bootstrap(k)) + let change + try { + await until(async () => reached) + finished = false + change = settle( + writer('certificates') + .where({ certificateId: 1 }) + .update({ userId: 2 }) + .then(() => { + finished = true + }) + ) + const locks = await until(() => waiting(writerId)) + assert.equal(finished, false) + assert.ok(locks.some(row => row.tableName === 'certificates')) + events.push({ case: 'bootstrap retains parent protection through key and cursor commit', locks }) + } finally { + release() + if (hadOwn) prototype.query = original + else delete prototype.query + success(await pending) + if (change !== undefined) success(await change) + } + await verify() + return { isolation, events } + } finally { + if (transaction && !transaction.isCompleted()) await transaction.rollback() + await observer.destroy() + await writer.destroy() + await k.destroy() + await admin.raw('DROP DATABASE ??', [database]) + } +} +async function qualifyMysqlCertificateIndexLocks(control, connection) { + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const results = [] + await runInSeries(['read committed', 'repeatable read'], async isolation => { + results.push(await fixture(control, connection, isolation)) + }) + return results +} +module.exports = { qualifyMysqlCertificateIndexLocks } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexMysqlSchedules.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexMysqlSchedules.cjs new file mode 100644 index 000000000..6877705c9 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexMysqlSchedules.cjs @@ -0,0 +1,171 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex } = require('knex') +const fc = require('fast-check') +const helper = require('../../out/src/storage/schema/snapshotCertificateIndexMigration.js') +const names = ['', 'A', 'a', 'é', 'e\u0301', 'Ω', '名字', '🚲', 'a ', 'A\0B', '😀'.repeat(100)] +async function oracle(k) { + const result = await k.raw( + 'SELECT userId,fieldName,certificateId,SUM(bit) AS membership FROM (SELECT userId,fieldName,certificateId,1 AS bit FROM certificate_fields UNION ALL SELECT c.userId,f.fieldName,f.certificateId,2 AS bit FROM certificate_fields f JOIN certificates c ON c.certificateId=f.certificateId) owned_fields GROUP BY userId,fieldName,certificateId ORDER BY userId,fieldName,certificateId' + ) + const expected = (k.client.config.client === 'mysql2' ? result[0] : result).map(row => ({ + ...row, + membership: Number(row.membership) + })) + const actual = await k('snapshot_certificate_field_keys') + .select({ + userId: 'snapshotUserId', + fieldName: 'snapshotFieldName', + certificateId: 'snapshotCertificateId', + membership: 'snapshotMembership' + }) + .orderBy(['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId']) + assert.deepEqual(actual, expected) +} +async function seed(k) { + await k('certificate_fields').delete() + await k('certificates').delete() + await k('certificates').insert([ + { certificateId: 1, userId: 1 }, + { certificateId: 2, userId: 2 } + ]) + await runInSeries(names.entries(), async ([i, fieldName]) => { + await k('certificate_fields') + .insert({ userId: (i % 2) + 1, fieldName, certificateId: (i % 2) + 1, fieldValue: 'initial' }) + .onConflict(['fieldName', 'certificateId']) + .merge(['userId', 'fieldValue']) + }) +} +async function operation(k, op, counts) { + counts[op.kind]++ + const fieldName = names[op.name], + certificateId = op.id + const selected = () => k('certificate_fields').where({ fieldName, certificateId }) + switch (op.kind) { + case 0: + await k('certificates').insert({ certificateId, userId: op.user }).onConflict('certificateId').merge(['userId']) + break + case 1: + await k('certificates').where({ certificateId }).delete() + break + case 2: + await k('certificate_fields') + .insert({ userId: op.user, fieldName, certificateId, fieldValue: 'value-' + op.user }) + .onConflict(['fieldName', 'certificateId']) + .merge(['userId', 'fieldValue']) + break + case 3: + await selected().delete() + break + case 4: + await selected().update({ userId: op.user }) + break + case 5: + if ( + !(await k('certificates') + .where({ certificateId: certificateId + 4 }) + .first()) + ) + await k('certificates') + .where({ certificateId }) + .update({ certificateId: certificateId + 4 }) + break + case 6: + try { + await selected().update({ fieldName: names[(op.name + 1) % names.length] }) + } catch (error) { + if (!['ER_DUP_ENTRY', 'SQLITE_CONSTRAINT_UNIQUE'].includes(error.code)) throw error + } + break + case 7: + await assert.rejects( + k.transaction(async trx => { + await trx('certificate_fields').where({ fieldName, certificateId }).delete() + throw new Error('synthetic operation rollback') + }), + /synthetic operation rollback/ + ) + break + case 8: + await k('certificates') + .where({ certificateId }) + .update({ isDeleted: k.raw('1-isDeleted') }) + break + } +} +async function qualify(k, collation) { + const mysql = k.client.config.client === 'mysql2' + await k.raw( + mysql + ? 'CREATE TABLE certificates(certificateId INT UNSIGNED PRIMARY KEY,userId INT UNSIGNED NOT NULL,isDeleted TINYINT NOT NULL DEFAULT 0) ENGINE=InnoDB' + : 'CREATE TABLE certificates(certificateId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,isDeleted INTEGER NOT NULL DEFAULT 0)' + ) + await k.raw( + mysql + ? 'CREATE TABLE certificate_fields(userId INT UNSIGNED NOT NULL,fieldName VARCHAR(100) NOT NULL,certificateId INT UNSIGNED NOT NULL,fieldValue VARCHAR(255) NOT NULL,UNIQUE(fieldName,certificateId)) ENGINE=InnoDB' + : `CREATE TABLE certificate_fields(userId INTEGER NOT NULL,fieldName VARCHAR(100) COLLATE ${collation} NOT NULL,certificateId INTEGER NOT NULL,fieldValue VARCHAR(255) NOT NULL,UNIQUE(fieldName,certificateId))` + ) + await seed(k) + assert.equal(await helper.readSnapshotCertificateIndexState(k), false) + await helper.addSnapshotCertificateIndexes(k) + await oracle(k) + await helper.addSnapshotCertificateIndexes(k) + await oracle(k) + assert.equal(await helper.readSnapshotCertificateIndexState(k), false) + await k.schema.createTable('knex_migrations', t => t.string('name')) + await k('knex_migrations').insert({ name: helper.SNAPSHOT_CERTIFICATE_INDEX_MIGRATION }) + assert.equal(await helper.readSnapshotCertificateIndexState(k), true) + const generated = fc.record({ + kind: fc.integer({ min: 0, max: 8 }), + id: fc.integer({ min: 1, max: 4 }), + user: fc.integer({ min: 1, max: 3 }), + name: fc.integer({ min: 0, max: names.length - 1 }) + }) + const counts = Array.from({ length: 9 }, () => 0) + await fc.assert( + fc.asyncProperty(fc.array(generated, { minLength: 1, maxLength: 24 }), async schedule => { + await seed(k) + await runInSeries(schedule, async op => { + await operation(k, op, counts) + await oracle(k) + }) + }), + { numRuns: 300, seed: 3242026 } + ) + const sourceRows = await k('certificate_fields').orderBy(['fieldName', 'certificateId']) + await helper.removeSnapshotCertificateIndexes(k) + assert.deepEqual(await k('certificate_fields').orderBy(['fieldName', 'certificateId']), sourceRows) + await assert.rejects(helper.readSnapshotCertificateIndexState(k), /incomplete/) + await helper.addSnapshotCertificateIndexes(k) + await oracle(k) + assert.equal(await helper.readSnapshotCertificateIndexState(k), true) + return { + dialect: mysql ? 'mysql' : 'sqlite', + collation, + runs: 300, + seed: 3242026, + counts, + repeatedInstall: true, + journalAdoption: true, + removalPreservesSource: true + } +} +async function qualifyMysqlCertificateIndexSchedules(control, connection) { + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const results = [] + await runInSeries(['utf8mb4_0900_ai_ci', 'utf8mb4_unicode_ci', 'utf8mb4_bin'], async collation => { + const database = 'ts569_certificate_schedules_' + randomUUID().replaceAll('-', '') + await control.raw(`CREATE DATABASE ?? CHARACTER SET utf8mb4 COLLATE ${collation}`, [database]) + const k = knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + try { + results.push(await qualify(k, collation)) + } finally { + await k.destroy() + await control.raw('DROP DATABASE ??', [database]) + } + }) + return results +} +module.exports = { oracle, qualifyMysqlCertificateIndexSchedules } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexSeeks.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexSeeks.cjs new file mode 100644 index 000000000..aa25e7857 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotCertificateIndexSeeks.cjs @@ -0,0 +1,200 @@ +// Invoked only by the verified disposable native MySQL fixture. +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + createKnexWalletSnapshotPageReader: pageReader +} = require('../../out/src/storage/snapshot/KnexWalletReadSnapshot.js') + +async function seedLargeFields(k, userId, otherId) { + const date = new Date('2026-01-01T00:00:00Z') + await runInSeries( + Array.from({ length: 32 }, (_, i) => i * 256), + async start => { + await k('certificate_fields').insert( + Array.from({ length: 256 }, (_, i) => { + const id = start + i + return { + created_at: date, + updated_at: date, + fieldName: String(id).padStart(6, '0'), + userId: id % 2 ? otherId : userId, + certificateId: id % 2 ? 2 : 1, + fieldValue: 'synthetic', + masterKey: 'synthetic' + } + }) + ) + } + ) +} + +async function handlerCounters(source, view) { + return await view.read(async trx => + Object.fromEntries( + (await source.toDb(trx).raw("SHOW SESSION STATUS LIKE 'Handler_read_%'"))[0].map(row => [ + row.Variable_name, + Number(row.Value) + ]) + ) + ) +} + +async function tableCounters(source, view) { + return await view.read(async trx => + Object.fromEntries( + ( + await source + .toDb(trx) + .raw( + 'SELECT OBJECT_NAME, COUNT_FETCH FROM performance_schema.table_io_waits_summary_by_table WHERE OBJECT_SCHEMA=DATABASE() ORDER BY OBJECT_NAME' + ) + )[0].map(row => [row.OBJECT_NAME, Number(row.COUNT_FETCH)]) + ) + ) +} + +async function observePage(source, view, read, leftId) { + const queries = [] + const listener = q => { + if (q.sql.startsWith('select') && q.sql.includes('snapshot_certificate_field_keys') && q.sql.includes('cross join')) + queries.push(q) + } + const tablesBefore = await tableCounters(source, view) + const before = await handlerCounters(source, view) + source.knex.on('query', listener) + let page + try { + page = await read( + 'certificateFields', + leftId === undefined + ? undefined + : { + version: 1, + snapshotId: 'large-seek-fixture', + table: 'certificateFields', + after: [String(leftId).padStart(6, '0'), 1] + }, + { maxRows: 16, maxBytes: 131072 } + ) + } finally { + source.knex.off('query', listener) + } + const after = await handlerCounters(source, view) + const tablesAfter = await tableCounters(source, view) + const fetches = Object.fromEntries( + Object.keys(tablesAfter).map(name => [name, tablesAfter[name] - (tablesBefore[name] ?? 0)]) + ) + const deltas = Object.fromEntries(Object.keys(after).map(key => [key, after[key] - before[key]])) + const expectedIds = Array.from( + { length: Math.min(16, (8190 - (leftId ?? -2)) / 2) }, + (_, i) => (leftId ?? -2) + (i + 1) * 2 + ) + assert.deepEqual( + page.rows.map(row => Number(row.fieldName)), + expectedIds + ) + const plans = [] + await runInSeries(queries, async q => { + plans.push(await view.read(async trx => (await source.toDb(trx).raw('EXPLAIN ' + q.sql, q.bindings))[0])) + }) + // Session Handler counters can include lazy work outside the wallet tables. + // Enforce native fetch bounds on this isolated database, including both page + // passes and ownership guards; retain session counters as diagnostic evidence. + assert.equal(fetches.certificate_fields, page.rows.length * 2) + assert.equal(fetches.certificates, page.rows.length) + assert.equal(fetches.users, 1) + assert.ok( + fetches.snapshot_certificate_field_keys <= page.rows.length * 2 + 4, + 'Both passes must seek bounded auxiliary keys: ' + JSON.stringify(fetches) + ) + assert.ok(fetches.snapshot_certificate_field_keys >= page.rows.length * 2) + const allowed = new Set(['certificate_fields', 'certificates', 'users', 'snapshot_certificate_field_keys']) + assert.ok( + Object.entries(fetches).every(([table, count]) => allowed.has(table) || count === 0), + 'A certificate page must not fetch unrelated wallet tables' + ) + assert.ok(deltas.Handler_read_rnd_next <= 64, 'The page must not scan rows into a temporary table') + assert.ok( + plans.every( + plan => + plan[0].table === 'snapshot_certificate_field_keys' && + plan[0].key === 'PRIMARY' && + (leftId === undefined ? ['ref', 'range'].includes(plan[0].type) : plan[0].type === 'range') + ), + 'Both page passes must use the profile prefix or cursor range' + ) + return { after: leftId ?? null, rows: page.rows.length, deltas, fetches, plans, queries } +} + +async function largeSeeks(source, userId, otherId) { + await seedLargeFields(source.knex, userId, otherId) + const results = [] + await runInSeries(['fresh', 'refreshed'], async statistics => { + if (statistics === 'refreshed') + await source.knex.raw('ANALYZE TABLE snapshot_certificate_field_keys, certificate_fields') + const view = await source.openReadSnapshot() + const read = pageReader(source, userId, 'large-seek-fixture', view, true, true, true) + try { + await runInSeries([undefined, 1500, 7000, 8180], async leftId => + results.push({ statistics, ...(await observePage(source, view, read, leftId)) }) + ) + } finally { + await view.close() + } + }) + return results +} + +async function qualifyCollation(control, connection, collation) { + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const database = 'ts569_certificate_seeks_' + randomUUID().replaceAll('-', '') + await control.raw(`CREATE DATABASE ?? CHARACTER SET utf8mb4 COLLATE ${collation}`, [database]) + const source = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + }) + try { + await source.migrate('synthetic certificate seek', 'synthetic-certificate-seek') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } + await runInSeries( + [ + [1, user.userId], + [2, other.userId] + ], + async ([id, userId]) => { + await source.knex('certificates').insert({ + ...dates, + certificateId: id, + userId, + serialNumber: 'certificate-' + id, + type: 'type', + certifier: '02' + '11'.repeat(32), + subject: '02' + '11'.repeat(32), + revocationOutpoint: 'a'.repeat(64) + '.0', + signature: 'synthetic', + isDeleted: false + }) + } + ) + return { collation, pages: await largeSeeks(source, user.userId, other.userId) } + } finally { + await source.destroy() + await control.raw('DROP DATABASE ??', [database]) + } +} +async function qualifyMysqlCertificateIndexSeeks(control, connection) { + const results = [] + await runInSeries(['utf8mb4_0900_ai_ci', 'utf8mb4_unicode_ci', 'utf8mb4_bin'], async collation => + results.push(await qualifyCollation(control, connection, collation)) + ) + return results +} +module.exports = { qualifyMysqlCertificateIndexSeeks } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexCrash.cjs new file mode 100644 index 000000000..2c5c60672 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexCrash.cjs @@ -0,0 +1,269 @@ +const { migrateBeforeSqliteGeneration } = require('./snapshotHistoricalMigrations.cjs') +// Synthetic process-loss qualification, invoked by the existing native fixtures. +const assert = require('node:assert/strict') +const { spawn } = require('node:child_process') +const { randomUUID } = require('node:crypto') +const fs = require('node:fs/promises') +const { writeFileSync } = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { knex, oracle } = require('./snapshotGlobalIndexFixtures.cjs') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + removeSnapshotGlobalIndexes, + readSnapshotGlobalIndexState, + SNAPSHOT_GLOBAL_INDEX_MIGRATION +} = require('../../out/src/storage/schema/snapshotGlobalIndexMigration.js') + +const phases = [ + 'after-guard-table', + 'after-key-table', + 'after-first-index', + 'partial-observers', + 'partial-producers', + 'before-cursor', + 'after-cursor', + 'after-commit' +] +const migrationName = 'synthetic global crash' +const migrationIdentity = 'synthetic-global-crash' + +const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } +const provider = options => + new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: knex(options) }) + +async function migrate(source) { + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) +} +async function seed(options) { + const source = provider(options) + try { + if (options.client === 'better-sqlite3') await source.knex.raw('PRAGMA journal_mode = WAL') + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + await source.makeAvailable() + await removeSnapshotGlobalIndexes(source.knex) + await source.knex('knex_migrations').where('name', SNAPSHOT_GLOBAL_INDEX_MIGRATION).delete() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await runInSeries([7, 9], async provenTxId => { + await source.knex('proven_txs').insert({ + ...dates, + provenTxId, + txid: String(provenTxId).repeat(64), + height: 1, + index: 0, + merklePath: Buffer.from([1]), + rawTx: Buffer.from([1]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + }) + await source.knex('proven_tx_reqs').insert({ + ...dates, + provenTxReqId: 5, + txid: 'a'.repeat(64), + provenTxId: 7, + status: 'unknown', + rawTx: Buffer.from([1]), + history: '{}', + notify: '{}' + }) + await runInSeries([0, 200, 400], async start => { + await source.knex('transactions').insert( + Array.from({ length: 200 }, (_, i) => ({ + ...dates, + transactionId: (start + i + 1) * 2, + userId: user.userId, + txid: 'a'.repeat(64), + provenTxId: (start + i) % 2 ? 7 : null, + status: 'completed', + reference: 'global-crash-' + (start + i), + isOutgoing: true, + satoshis: 0, + description: '' + })) + ) + }) + return { userId: user.userId, otherId: other.userId } + } finally { + await source.destroy() + } +} + +async function child(options, phase, marker) { + assert(phases.includes(phase)) + assert.equal(typeof process.send, 'function', 'Child requires its fixture parent') + const source = provider(options) + let cursorWritten = false + const cursor = query => + query.sql.startsWith('update `snapshot_global_index_progress`') && query.bindings.includes(512) + const park = event => { + writeFileSync(marker, JSON.stringify({ phase, event }), { mode: 0o600 }) + process.kill(process.pid, 'SIGKILL') + } + source.knex.on('query', query => { + if (phase === 'before-cursor' && cursor(query)) park('query') + if (phase === 'after-commit' && cursorWritten && query.sql.toLowerCase().startsWith('begin')) + park('next-transaction') + }) + source.knex.on('query-response', (_result, query) => { + const sql = query.sql.toLowerCase() + if (phase === 'after-guard-table' && sql.startsWith('create table `snapshot_global_guards`')) park('query-response') + if (phase === 'after-key-table' && sql.startsWith('create table `snapshot_global_keys`')) park('query-response') + if ( + phase === 'after-first-index' && + (sql.startsWith('create index `snapshot_global_page`') || + sql.startsWith('alter table `snapshot_global_keys` add index `snapshot_global_page`')) + ) + park('query-response') + if (phase === 'partial-observers' && sql.startsWith('create trigger snapshot_global_edge_delete ')) + park('query-response') + if (phase === 'partial-producers' && sql.startsWith('create trigger snapshot_global_tx_insert ')) + park('query-response') + if (cursor(query)) { + cursorWritten = true + if (phase === 'after-cursor') park('query-response') + } + if (phase === 'after-commit' && cursorWritten && sql.startsWith('commit')) park('query-response') + }) + try { + await migrate(source) + throw new Error('Expected migration boundary was not reached') + } finally { + await source.destroy() + } +} + +async function terminateAt(options, phase) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-global-boundary-')) + const marker = path.join(directory, 'boundary.json') + const processHandle = spawn(process.execPath, [__filename, 'child'], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + let stderr = '' + processHandle.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-65536) + }) + const exited = new Promise((resolve, reject) => { + processHandle.once('exit', (code, signal) => resolve({ code, signal })) + processHandle.once('error', reject) + }) + const timer = setTimeout(() => processHandle.kill('SIGKILL'), 15000) + try { + processHandle.send({ options, phase, marker }) + const result = await exited + assert.equal(result.signal, 'SIGKILL', stderr) + const observed = JSON.parse(await fs.readFile(marker, 'utf8')) + assert.equal(observed.phase, phase) + return { phase, event: observed.event, signal: result.signal } + } finally { + clearTimeout(timer) + if (processHandle.exitCode === null && processHandle.signalCode === null) { + processHandle.kill('SIGKILL') + await exited + } + await fs.rm(directory, { recursive: true, force: true }) + } +} + +async function qualify(options, phase) { + const { userId, otherId } = await seed(options) + const outcome = await terminateAt(options, phase) + const source = provider(options) + const database = source.knex + try { + assert.equal(await readSnapshotGlobalIndexState(database), false) + if (['before-cursor', 'after-cursor', 'after-commit'].includes(phase)) { + const committed = phase === 'after-commit' + const progress = await database('snapshot_global_index_progress').where('id', 0).first() + assert.equal(progress.afterRowId, committed ? 512 : 0) + assert.equal( + Number((await database('snapshot_global_edges').count({ count: '*' }).first()).count), + committed ? 640 : 0 + ) + } + await database('transactions').where('transactionId', 2).update({ userId: otherId }) + await database('transactions').where('transactionId', 4).delete() + await database('transactions').insert({ + ...dates, + transactionId: 3, + userId, + txid: 'a'.repeat(64), + provenTxId: null, + status: 'completed', + reference: 'behind-cursor', + isOutgoing: true, + satoshis: 0, + description: '' + }) + await database('proven_tx_reqs').where('provenTxReqId', 5).update({ provenTxId: 9 }) + // A killed migrator leaves Knex's lock claimed. This is fixture-owned recovery; + // the verified child is gone and no other migrator can own this isolated store. + await database.migrate.forceFreeMigrationsLock() + await migrate(source) + assert.equal(await readSnapshotGlobalIndexState(database), true) + await oracle(database) + assert.equal( + Number( + (await database('knex_migrations').where('name', SNAPSHOT_GLOBAL_INDEX_MIGRATION).count({ count: '*' }).first()) + .count + ), + 1 + ) + return { ...outcome, journalPublishedAfterRecovery: true, independentProfileChangeAndLowIdInsert: true } + } finally { + await source.destroy() + } +} + +async function qualifySQLiteGlobalIndexProcessLoss() { + const results = [] + await runInSeries(phases, async phase => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-global-index-crash-')) + try { + results.push( + await qualify( + { + client: 'better-sqlite3', + connection: { filename: path.join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }, + phase + ) + ) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + return results +} + +async function qualifyMysqlGlobalIndexProcessLoss(control, connection) { + // The calling fixture has already verified its disposable, pinned container. + const results = [] + await runInSeries(phases, async phase => { + const database = 'ts569_global_' + randomUUID().replaceAll('-', '') + await control.raw('CREATE DATABASE ??', [database]) + try { + results.push( + await qualify({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }, phase) + ) + } finally { + await control.raw('DROP DATABASE ??', [database]) + } + }) + return results +} + +if (process.argv[2] === 'child') { + assert.equal(typeof process.send, 'function', 'Child execution requires its fixture parent') + process.once('message', ({ options, phase, marker }) => { + child(options, phase, marker).catch(error => { + console.error(error) + process.exitCode = 1 + process.disconnect() + }) + }) +} +module.exports = { qualifySQLiteGlobalIndexProcessLoss, qualifyMysqlGlobalIndexProcessLoss } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexFixtures.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexFixtures.cjs new file mode 100644 index 000000000..c8b75efec --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexFixtures.cjs @@ -0,0 +1,171 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const assert = require('node:assert/strict') +const { knex } = require('knex') +const fc = require('fast-check') +const edges = 'snapshot_global_edges', + keys = 'snapshot_global_keys', + guards = 'snapshot_global_guards' +async function sourceSchema(k, collation) { + const mysql = k.client.config.client === 'mysql2' + const integer = mysql ? 'INT UNSIGNED' : 'INTEGER' + const suffix = mysql ? ' ENGINE=InnoDB' : '' + const text = `VARCHAR(64) COLLATE ${collation}` + await k.raw(`CREATE TABLE proven_txs(provenTxId ${integer} NOT NULL PRIMARY KEY)${suffix}`) + await k.raw( + `CREATE TABLE proven_tx_reqs(provenTxReqId ${integer} NOT NULL PRIMARY KEY,provenTxId ${integer},txid ${text} NOT NULL UNIQUE)${suffix}` + ) + await k.raw( + `CREATE TABLE transactions(transactionId ${integer} NOT NULL PRIMARY KEY,userId ${integer} NOT NULL,provenTxId ${integer},txid ${text})${suffix}` + ) + await k.schema.alterTable('transactions', t => t.index('txid')) +} +async function oracle(k) { + const direct = k('transactions') + .select(k.raw('transactionId,0 AS requestId,1 AS tableId,provenTxId AS rowId,userId')) + .whereNotNull('provenTxId') + const req = k('transactions AS t') + .join('proven_tx_reqs AS r', 'r.txid', 't.txid') + .select(k.raw('t.transactionId,r.provenTxReqId AS requestId,0 AS tableId,r.provenTxReqId AS rowId,t.userId')) + const indirect = k('transactions AS t') + .join('proven_tx_reqs AS r', 'r.txid', 't.txid') + .select(k.raw('t.transactionId,r.provenTxReqId AS requestId,1 AS tableId,r.provenTxId AS rowId,t.userId')) + .whereNotNull('r.provenTxId') + const expected = [...(await direct), ...(await req), ...(await indirect)] + const order = rows => + rows + .map(row => [row.transactionId, row.requestId, row.tableId, row.rowId, row.userId]) + .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))) + assert.deepEqual(order(await k(edges)), order(expected), 'Independent relational union must equal every edge') + const proofs = new Set((await k('proven_txs')).map(row => row.provenTxId)) + const counts = new Map() + for (const row of expected) { + const key = JSON.stringify([row.tableId, row.userId, row.rowId]) + counts.set(key, (counts.get(key) ?? 0) + 1) + } + const actual = await k(keys) + assert.equal(actual.length, counts.size) + for (const row of actual) { + const key = JSON.stringify([row.tableId, row.userId, row.rowId]) + assert.equal(row.refs, counts.get(key), key) + assert.equal(Number(row.present), row.tableId === 0 || proofs.has(row.rowId) ? 1 : 0, key) + } + const expectedGuards = [...new Set(expected.filter(row => row.tableId === 1).map(row => row.rowId))].sort( + (a, b) => a - b + ) + const actualGuards = await k(guards).orderBy('proofId') + assert.deepEqual( + actualGuards.map(row => row.proofId), + expectedGuards, + 'No unused proof serialization rows' + ) + for (const row of actualGuards) assert.equal(Number(row.present), proofs.has(row.proofId) ? 1 : 0) + return { edges: expected.length, keys: counts.size } +} +async function sequential(k) { + const records = [] + const step = async (name, action) => { + await action() + records.push({ name, ...(await oracle(k)) }) + } + await step('proof', () => k('proven_txs').insert({ provenTxId: 7 })) + await step('direct owner', () => k('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: 7 })) + await step('duplicate owner', () => + k('transactions').insert({ transactionId: 2, userId: 1, txid: 'a', provenTxId: 7 }) + ) + await step('other profile', () => k('transactions').insert({ transactionId: 3, userId: 2, txid: 'a', provenTxId: 7 })) + await step('same proof through request', () => + k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: 7 }) + ) + await step('one reference removed', () => k('transactions').where('transactionId', 1).delete()) + await step('proof absent with references', () => k('proven_txs').where('provenTxId', 7).delete()) + await step('proof returned', () => k('proven_txs').insert({ provenTxId: 7 })) + await step('new absent indirect proof', () => k('proven_tx_reqs').where('provenTxReqId', 5).update({ provenTxId: 9 })) + await step('new proof appears', () => k('proven_txs').insert({ provenTxId: 9 })) + await step('transaction moves profile', () => k('transactions').where('transactionId', 2).update({ userId: 3 })) + await step('request changes txid', () => k('proven_tx_reqs').where('provenTxReqId', 5).update({ txid: 'b' })) + await step('transaction matches new txid', () => k('transactions').where('transactionId', 3).update({ txid: 'b' })) + await step('request changes ID', () => k('proven_tx_reqs').where('provenTxReqId', 5).update({ provenTxReqId: 6 })) + await step('transaction changes ID', () => k('transactions').where('transactionId', 2).update({ transactionId: 20 })) + await step('proof changes ID', () => k('proven_txs').where('provenTxId', 9).update({ provenTxId: 10 })) + await step('request removed', () => k('proven_tx_reqs').where('provenTxReqId', 6).delete()) + await step('last direct references removed', () => k('transactions').delete()) + return records +} +async function generated(k) { + let schedules = 0, + steps = 0, + expectedConstraintFailures = 0 + const op = fc.record({ + kind: fc.integer({ min: 0, max: 8 }), + id: fc.integer({ min: 1, max: 5 }), + userId: fc.integer({ min: 1, max: 3 }), + otherId: fc.integer({ min: 1, max: 5 }), + nullable: fc.boolean() + }) + await fc.assert( + fc.asyncProperty(fc.array(op, { minLength: 1, maxLength: 24 }), async ops => { + await k('transactions').delete() + await k('proven_tx_reqs').delete() + await k('proven_txs').delete() + await oracle(k) + await runInSeries(ops, async o => { + const proof = o.nullable ? null : o.otherId, + txid = o.nullable ? null : 'r' + o.otherId + try { + switch (o.kind) { + case 0: + await k('transactions') + .insert({ transactionId: o.id, userId: o.userId, txid, provenTxId: proof }) + .onConflict('transactionId') + .merge() + break + case 1: + await k('transactions').where('transactionId', o.id).delete() + break + case 2: + await k('transactions').where('transactionId', o.id).update({ userId: o.userId, txid, provenTxId: proof }) + break + case 3: + await k('proven_tx_reqs') + .insert({ provenTxReqId: o.id, txid: 'r' + o.id, provenTxId: proof }) + .onConflict('provenTxReqId') + .merge() + break + case 4: + await k('proven_tx_reqs').where('provenTxReqId', o.id).delete() + break + case 5: + await k('proven_tx_reqs') + .where('provenTxReqId', o.id) + .update({ txid: 'r' + o.otherId, provenTxId: proof }) + break + case 6: + await k('proven_txs').insert({ provenTxId: o.id }).onConflict('provenTxId').ignore() + break + case 7: + await k('proven_txs').where('provenTxId', o.id).delete() + break + case 8: + await k('proven_txs').where('provenTxId', o.id).update({ provenTxId: o.otherId }) + break + } + } catch (error) { + if ( + error.code === 'ER_DUP_ENTRY' || + error.code === 'SQLITE_CONSTRAINT_PRIMARYKEY' || + error.code === 'SQLITE_CONSTRAINT_UNIQUE' + ) + expectedConstraintFailures++ + else throw error + } + await oracle(k) + steps++ + }) + schedules++ + }), + { numRuns: 300, seed: 3242026, endOnFailure: true } + ) + assert.equal(schedules, 300) + return { schedules, steps, expectedConstraintFailures, seed: 3242026 } +} +module.exports = { oracle, sequential, sourceSchema, generated, knex } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexIntegration.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexIntegration.cjs new file mode 100644 index 000000000..b8ca4eba0 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexIntegration.cjs @@ -0,0 +1,320 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex, oracle, sourceSchema } = require('./snapshotGlobalIndexFixtures.cjs') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const helper = require('../../out/src/storage/schema/snapshotGlobalIndexMigration.js') +const { + createKnexWalletSnapshotPageReader: pageReader, + openKnexWalletReadSnapshot +} = require('../../out/src/storage/snapshot/KnexWalletReadSnapshot.js') +const { openKnexSnapshotArchiveSource } = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveSource.js') +const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } +const identity = '02' + '11'.repeat(32) +const standardKeys = { proven_txs: 'provenTxId', proven_tx_reqs: 'provenTxReqId', transactions: 'transactionId' } +async function compareReaders(source, users) { + const tables = [ + 'provenTxs', + 'provenTxReqs', + 'outputBaskets', + 'transactions', + 'commissions', + 'outputs', + 'outputTags', + 'outputTagMaps', + 'txLabels', + 'txLabelMaps', + 'certificates', + 'certificateFields', + 'syncStates' + ] + const results = [] + await runInSeries(users, async userId => { + const view = await source.openReadSnapshot() + try { + const legacy = pageReader(source, userId, 'global-draft', view, true, true, true, false) + const indexed = pageReader(source, userId, 'global-draft', view, true, true, true, true) + await runInSeries(tables, async table => { + let cursor, + rows = 0, + pages = 0, + complete = false + function* pendingPages() { + while (!complete) yield undefined + } + await runInSeries(pendingPages(), async () => { + const options = { maxRows: 17, maxBytes: 131072 } + const expected = await legacy(table, cursor, options) + const actual = await indexed(table, cursor, options) + assert.deepEqual(actual, expected, `All source rows, public cursor and byte charges must match for ${table}`) + rows += actual.rows.length + pages++ + assert.ok(pages < 1000) + complete = actual.done + cursor = actual.cursor + }) + results.push({ userId, table, rows, pages }) + }) + } finally { + await view.close() + } + }) + return results +} +async function qualifySchema(options) { + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: knex(options) }) + const k = source.knex + try { + await source.migrate('synthetic global draft', 'synthetic-global-draft') + await source.makeAvailable() + const users = [] + await runInSeries(['02' + '11'.repeat(32), '03' + '22'.repeat(32)], async identity => { + users.push((await source.findOrInsertUser(identity)).user.userId) + }) + await runInSeries([7, 9], async provenTxId => { + await k('proven_txs').insert({ + ...dates, + provenTxId, + txid: String(provenTxId).repeat(64), + height: 1, + index: 0, + merklePath: Buffer.from([1]), + rawTx: Buffer.from([1]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + }) + await runInSeries([5, 6], async provenTxReqId => { + await k('proven_tx_reqs').insert({ + ...dates, + provenTxReqId, + txid: String(provenTxReqId).repeat(64), + provenTxId: 9, + status: 'unknown', + history: '{}', + notify: '{}', + rawTx: Buffer.from([1]) + }) + }) + await runInSeries([0, 1, 2], async batch => { + await k('transactions').insert( + Array.from({ length: 200 }, (_, i) => ({ + ...dates, + transactionId: batch * 200 + i + 1, + userId: users[i % 2], + txid: String(i % 2 ? 5 : 6).repeat(64), + provenTxId: i % 2 ? 7 : null, + status: 'completed', + reference: 'global-' + (batch * 200 + i), + isOutgoing: true, + satoshis: 0, + description: '' + })) + ) + }) + const oldIndexes = {} + await runInSeries(Object.keys(standardKeys), async table => { + oldIndexes[table] = + options.client === 'mysql2' + ? (await k.raw('SHOW INDEXES FROM ??', [table]))[0].map(row => [ + row.Key_name, + row.Seq_in_index, + row.Column_name + ]) + : await k.raw('PRAGMA index_list(??)', [table]) + }) + await helper.addSnapshotGlobalIndexes(k) + const initial = await oracle(k) + const initialReaders = await compareReaders(source, users) + await k('transactions').where('transactionId', 1).update({ userId: users[1], provenTxId: 7 }) + await oracle(k) + await k('proven_tx_reqs').where('provenTxReqId', 5).update({ provenTxId: 7 }) + await oracle(k) + await k('transactions').whereIn('transactionId', [3, 7, 19, 21]).delete() + await oracle(k) + const changedReaders = await compareReaders(source, users) + const standard = {} + await runInSeries(Object.keys(standardKeys), async table => { + standard[table] = await k(table).orderBy(standardKeys[table]) + const now = + options.client === 'mysql2' + ? (await k.raw('SHOW INDEXES FROM ??', [table]))[0].map(row => [ + row.Key_name, + row.Seq_in_index, + row.Column_name + ]) + : await k.raw('PRAGMA index_list(??)', [table]) + assert.deepEqual(now, oldIndexes[table]) + }) + await helper.removeSnapshotGlobalIndexes(k) + await runInSeries(Object.keys(standard), async table => { + assert.deepEqual(await k(table).orderBy(standardKeys[table]), standard[table]) + }) + await helper.addSnapshotGlobalIndexes(k) + const reinstalled = await oracle(k) + return { + realStorageKnexSchema: true, + dialect: options.client, + bootstrapRows: 600, + standardIndexesPreserved: true, + sourceRowsPreservedOnRemoval: true, + initial, + reinstalled, + initialReaders, + changedReaders + } + } finally { + await source.destroy() + } +} +async function qualifyPinned(options, archive) { + const open = archive ? openKnexSnapshotArchiveSource : openKnexWalletReadSnapshot + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: knex(options) }) + const writer = knex(options), + k = source.knex + let before, after + try { + if (options.client === 'better-sqlite3') await k.raw('PRAGMA journal_mode = WAL') + await source.migrate('global pinned draft', 'global-pinned-draft') + await source.makeAvailable() + const { user } = await source.findOrInsertUser(identity) + await runInSeries([7, 9], async provenTxId => { + await k('proven_txs').insert({ + ...dates, + provenTxId, + txid: String(provenTxId).repeat(64), + height: 1, + index: 0, + merklePath: Buffer.from([1]), + rawTx: Buffer.from([1]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + }) + await k('proven_tx_reqs').insert({ + ...dates, + provenTxReqId: 5, + txid: 'a'.repeat(64), + provenTxId: 7, + status: 'unknown', + rawTx: Buffer.from([1]), + history: '{}', + notify: '{}' + }) + await k('transactions').insert({ + ...dates, + transactionId: 1, + userId: user.userId, + txid: 'a'.repeat(64), + provenTxId: 7, + status: 'completed', + reference: 'pinned-global', + isOutgoing: true, + satoshis: 0, + description: '' + }) + const queries = [] + k.on('query', q => { + if (q.sql.includes('cross join')) queries.push(q.sql) + }) + before = await open(source, identity, {}) + await writer.transaction(async trx => { + await trx('transactions').where('transactionId', 1).update({ provenTxId: 9 }) + await trx('proven_tx_reqs').where('provenTxReqId', 5).update({ provenTxId: 9 }) + }) + await oracle(writer) + if (archive) await before.validateClosure() + assert.deepEqual( + (await before.readPage('provenTxs')).rows.map(row => row.provenTxId), + [7] + ) + assert.deepEqual( + (await before.readPage('provenTxReqs')).rows.map(row => row.provenTxId), + [7] + ) + assert.ok(queries.some(sql => sql.includes('snapshot_global_keys'))) + await before.close() + before = undefined + after = await open(source, identity, {}) + assert.deepEqual( + (await after.readPage('provenTxs')).rows.map(row => row.provenTxId), + [9] + ) + assert.deepEqual( + (await after.readPage('provenTxReqs')).rows.map(row => row.provenTxId), + [9] + ) + await writer('transactions').where('transactionId', 1).delete() + assert.deepEqual( + (await after.readPage('provenTxs')).rows.map(row => row.provenTxId), + [9] + ) + await after.close() + after = undefined + after = await open(source, identity, {}) + assert.equal((await after.readPage('provenTxs')).rows.length, 0) + assert.equal((await after.readPage('provenTxReqs')).rows.length, 0) + return { + archive, + dialect: options.client, + automaticJournalAdoption: true, + independentWriterCommittedWhileOpen: true, + retainedOldReference: true, + freshViewNewReference: true, + finalReferenceRemoval: true + } + } finally { + if (before) await before.close() + if (after) await after.close() + await writer.destroy() + await source.destroy() + } +} +async function qualifyCascade(options) { + const k = knex(options) + try { + await sourceSchema(k, 'utf8mb4_0900_ai_ci') + await k.raw( + 'ALTER TABLE transactions ADD CONSTRAINT synthetic_cascade FOREIGN KEY(provenTxId) REFERENCES proven_txs(provenTxId) ON DELETE SET NULL' + ) + await assert.rejects(helper.addSnapshotGlobalIndexes(k), /requires explicit row mutations/) + assert.equal(await k.schema.hasTable('snapshot_global_guards'), false) + await k.raw('ALTER TABLE transactions DROP FOREIGN KEY synthetic_cascade') + await k.raw( + 'ALTER TABLE transactions ADD CONSTRAINT synthetic_restrict FOREIGN KEY(provenTxId) REFERENCES proven_txs(provenTxId) ON DELETE RESTRICT ON UPDATE RESTRICT' + ) + await helper.addSnapshotGlobalIndexes(k) + await k('proven_txs').insert({ provenTxId: 7 }) + await k('transactions').insert({ transactionId: 1, userId: 1, provenTxId: 7, txid: null }) + await oracle(k) + return { unsupportedCascadeRejectedBeforeDDL: true, standardRestrictSupported: true } + } finally { + await k.destroy() + } +} +async function qualifyMysqlGlobalIndexIntegration(admin, connection) { + const results = [] + await runInSeries( + [ + ['schema', qualifySchema], + ['ordinary', options => qualifyPinned(options, false)], + ['archive', options => qualifyPinned(options, true)], + ['cascade', qualifyCascade] + ], + async ([name, qualify]) => { + const database = 'ts569_global_integration_' + randomUUID().replaceAll('-', '') + await admin.raw('CREATE DATABASE ??', [database]) + try { + results.push({ + name, + ...(await qualify({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } })) + }) + } finally { + await admin.raw('DROP DATABASE ??', [database]) + } + } + ) + return results +} +module.exports = { qualifyMysqlGlobalIndexIntegration } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexMysql.cjs new file mode 100644 index 000000000..7ca6da8fc --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexMysql.cjs @@ -0,0 +1,136 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +// Native current-read and exact reference-count concurrency qualification. +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { oracle, knex, sourceSchema } = require('./snapshotGlobalIndexFixtures.cjs') +const { addSnapshotGlobalIndexes } = require('../../out/src/storage/schema/snapshotGlobalIndexMigration.js') +const settle = p => + p.then( + value => ({ ok: true, value }), + error => ({ ok: false, error }) + ) +const success = result => { + if (!result.ok) throw result.error +} +async function cases(control, connection, isolation) { + const database = 'ts569_global_locks_' + randomUUID().replaceAll('-', '') + await control.raw('CREATE DATABASE ??', [database]) + const open = () => knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + const k = open(), + writer = open(), + observer = open(), + results = [] + let trx + try { + await runInSeries([k, writer], async db => { + await db.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation.toUpperCase()) + }) + await sourceSchema(k, 'utf8mb4_0900_ai_ci') + await addSnapshotGlobalIndexes(k) + const requester = Number((await k.raw('SELECT CONNECTION_ID() AS id'))[0][0].id) + const wait = async () => { + let observed + function* pending() { + for (let i = 0; i < 1000 && !observed; i++) yield i + } + await runInSeries(pending(), async () => { + const [rows] = await observer.raw( + 'SELECT l.OBJECT_NAME AS tableName,l.LOCK_MODE AS lockMode FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE JOIN performance_schema.threads t ON t.THREAD_ID=l.THREAD_ID WHERE l.OBJECT_SCHEMA=DATABASE() AND t.PROCESSLIST_ID=?', + [requester] + ) + if (rows.length) observed = rows + else await new Promise(resolve => setTimeout(resolve, 5)) + }) + if (observed) return observed + throw new Error('No native lock observed within five seconds') + } + const reset = async () => { + await k('transactions').delete() + await k('proven_tx_reqs').delete() + await k('proven_txs').delete() + await oracle(k) + } + const locked = async (name, held, pending) => { + trx = await writer.transaction() + let operation + try { + await held(trx) + operation = settle(pending()) + const locks = await wait() + await trx.commit() + success(await operation) + results.push({ name, locks, ...(await oracle(k)) }) + } finally { + if (!trx.isCompleted()) await trx.rollback() + if (operation) success(await operation) + } + } + await locked( + 'proof insertion waits for absent-proof membership publication', + t => t('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: 71 }), + () => k('proven_txs').insert({ provenTxId: 71 }) + ) + await reset() + await locked( + 'membership waits for proof insertion', + t => t('proven_txs').insert({ provenTxId: 71 }), + () => k('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: 71 }) + ) + await reset() + await k('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: 71 }) + trx = await writer.transaction() + try { + assert.equal(Number((await trx('snapshot_global_guards').where('proofId', 71).first()).present), 0) + await k('proven_txs').insert({ provenTxId: 71 }) + await trx('transactions').insert({ transactionId: 2, userId: 2, txid: 'b', provenTxId: 71 }) + await trx.commit() + results.push({ name: 'old snapshot cannot publish stale proof presence', ...(await oracle(k)) }) + } finally { + if (!trx.isCompleted()) await trx.rollback() + } + await reset() + await k('proven_txs').insert([{ provenTxId: 71 }, { provenTxId: 72 }]) + await k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: 71 }) + trx = await writer.transaction() + try { + assert.equal((await trx('proven_tx_reqs').where('provenTxReqId', 5).first()).provenTxId, 71) + await k('proven_tx_reqs').where('provenTxReqId', 5).update({ provenTxId: 72 }) + await trx('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: null }) + await trx.commit() + results.push({ name: 'old snapshot cannot restore old request proof', ...(await oracle(k)) }) + } finally { + if (!trx.isCompleted()) await trx.rollback() + } + await reset() + await k('proven_txs').insert({ provenTxId: 71 }) + await k('transactions').insert([ + { transactionId: 1, userId: 1, txid: 'a', provenTxId: 71 }, + { transactionId: 2, userId: 1, txid: 'b', provenTxId: 71 } + ]) + await locked( + 'concurrent final reference removals preserve exact count', + t => t('transactions').where('transactionId', 1).delete(), + () => k('transactions').where('transactionId', 2).delete() + ) + await reset() + await k('proven_txs').insert({ provenTxId: 71 }) + await locked( + 'request insertion sees pending matching transaction', + t => t('transactions').insert({ transactionId: 1, userId: 1, txid: 'a', provenTxId: null }), + () => k('proven_tx_reqs').insert({ provenTxReqId: 5, txid: 'a', provenTxId: 71 }) + ) + return { isolation, results } + } finally { + if (trx && !trx.isCompleted()) await trx.rollback() + await Promise.all([k.destroy(), writer.destroy(), observer.destroy()]) + await control.raw('DROP DATABASE ??', [database]) + } +} +async function qualifyMysqlGlobalIndexLocks(control, connection) { + const results = [] + await runInSeries(['read committed', 'repeatable read'], async isolation => { + results.push(await cases(control, connection, isolation)) + }) + return results +} +module.exports = { qualifyMysqlGlobalIndexLocks } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexMysqlSchedules.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexMysqlSchedules.cjs new file mode 100644 index 000000000..535e736d9 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexMysqlSchedules.cjs @@ -0,0 +1,102 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex, oracle, sequential, sourceSchema, generated } = require('./snapshotGlobalIndexFixtures.cjs') +const helper = require('../../out/src/storage/schema/snapshotGlobalIndexMigration.js') +async function qualifySchedules(k, collation) { + await sourceSchema(k, collation) + await k('proven_txs').insert([{ provenTxId: 7 }, { provenTxId: 9 }]) + await k('proven_tx_reqs').insert([ + { provenTxReqId: 5, txid: 'a', provenTxId: 9 }, + { provenTxReqId: 6, txid: 'b', provenTxId: 7 } + ]) + await runInSeries([0, 200, 400], async start => { + await k('transactions').insert( + Array.from({ length: 200 }, (_, i) => ({ + transactionId: start + i + 1, + userId: ((start + i) % 3) + 1, + txid: (start + i) % 2 ? 'a' : 'b', + provenTxId: (start + i) % 2 ? 7 : null + })) + ) + }) + assert.equal(await helper.readSnapshotGlobalIndexState(k), false) + await helper.addSnapshotGlobalIndexes(k) + const bootstrap = await oracle(k) + const before = await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId']) + await helper.addSnapshotGlobalIndexes(k) + assert.deepEqual(await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId']), before) + await k('snapshot_global_index_progress').where('id', 0).update({ afterRowId: 0, complete: false }) + await helper.addSnapshotGlobalIndexes(k) + assert.deepEqual(await k('snapshot_global_keys').orderBy(['tableId', 'userId', 'rowId']), before) + assert.equal(await helper.readSnapshotGlobalIndexState(k), false) + await k.schema.createTable('knex_migrations', t => t.string('name')) + await k('knex_migrations').insert({ name: helper.SNAPSHOT_GLOBAL_INDEX_MIGRATION }) + assert.equal(await helper.readSnapshotGlobalIndexState(k), true) + await k('transactions').delete() + await k('proven_tx_reqs').delete() + await k('proven_txs').delete() + const transitions = await sequential(k) + const schedules = await generated(k) + const source = await k('transactions').orderBy('transactionId') + await helper.removeSnapshotGlobalIndexes(k) + assert.deepEqual(await k('transactions').orderBy('transactionId'), source) + await assert.rejects(helper.readSnapshotGlobalIndexState(k), /incomplete/) + await helper.addSnapshotGlobalIndexes(k) + await oracle(k) + assert.equal(await helper.readSnapshotGlobalIndexState(k), true) + return { + dialect: k.client.config.client, + collation, + bootstrap, + bootstrapRows: 600, + repeatedInstall: true, + resumedReplay: true, + journalAdoption: true, + removalPreservesSource: true, + sequential: transitions.length, + ...schedules + } +} +async function qualifyBoundaries(k, collation) { + await sourceSchema(k, collation) + await helper.addSnapshotGlobalIndexes(k) + await k('proven_txs').insert({ provenTxId: 4294967295 }) + await k('proven_tx_reqs').insert({ provenTxReqId: 4294967294, txid: 'A', provenTxId: 4294967295 }) + await k('transactions').insert({ transactionId: 4294967293, userId: 4294967292, txid: 'A', provenTxId: 4294967295 }) + await oracle(k) + await runInSeries(['a', 'a ', 'é', 'e\u0301', 'A\0B', '😀'.repeat(64)], async text => { + await k('proven_tx_reqs').where('provenTxReqId', 4294967294).update({ txid: text }) + await oracle(k) + await k('transactions').where('transactionId', 4294967293).update({ txid: text }) + await oracle(k) + }) + await k('snapshot_global_index_progress').where('id', 0).update({ afterRowId: 0, complete: false }) + await helper.addSnapshotGlobalIndexes(k) + await oracle(k) + return { collation, unsignedIdBoundary: 4294967295, txidTextAndReplay: true } +} +async function qualifyMysqlGlobalIndexSchedules(admin, connection) { + const results = [] + await runInSeries(['utf8mb4_0900_ai_ci', 'utf8mb4_unicode_ci', 'utf8mb4_bin'], async collation => { + await runInSeries( + [ + ['schedules', qualifySchedules], + ['boundaries', qualifyBoundaries] + ], + async ([name, qualify]) => { + const database = 'ts569_global_' + randomUUID().replaceAll('-', '') + await admin.raw(`CREATE DATABASE ?? CHARACTER SET utf8mb4 COLLATE ${collation}`, [database]) + const k = knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + try { + results.push({ name, ...(await qualify(k, collation)) }) + } finally { + await k.destroy() + await admin.raw('DROP DATABASE ??', [database]) + } + } + ) + }) + return results +} +module.exports = { qualifyMysqlGlobalIndexSchedules } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexSeeks.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexSeeks.cjs new file mode 100644 index 000000000..70cbf6ec4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotGlobalIndexSeeks.cjs @@ -0,0 +1,163 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { addSnapshotGlobalIndexes } = require('../../out/src/storage/schema/snapshotGlobalIndexMigration.js') +const { + createKnexWalletSnapshotPageReader: pageReader +} = require('../../out/src/storage/snapshot/KnexWalletReadSnapshot.js') +const date = new Date('2026-01-01'), + dates = { created_at: date, updated_at: date } +const txid = id => id.toString(16).padStart(64, '0') +async function seed(source) { + const k = source.knex + const users = [] + await runInSeries(['02' + '11'.repeat(32), '03' + '22'.repeat(32)], async key => { + users.push((await source.findOrInsertUser(key)).user.userId) + }) + await runInSeries( + Array.from({ length: 32 }, (_, i) => i * 256), + async start => { + const ids = Array.from({ length: 256 }, (_, i) => start + i + 1) + await k('proven_txs').insert( + ids.map(id => ({ + ...dates, + provenTxId: id, + txid: txid(id), + height: 1, + index: 0, + merklePath: Buffer.from([1]), + rawTx: Buffer.from([1]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + })) + ) + await k('proven_tx_reqs').insert( + ids.map(id => ({ + ...dates, + provenTxReqId: id, + provenTxId: id, + txid: txid(id), + history: '{}', + notify: '{}', + status: 'unknown', + rawTx: Buffer.from([1]) + })) + ) + await k('transactions').insert( + ids.map(id => ({ + ...dates, + transactionId: id, + userId: users[(id - 1) % 2], + txid: txid(id), + provenTxId: id, + status: 'completed', + reference: 'global-seek-' + id, + isOutgoing: true, + satoshis: 0, + description: '' + })) + ) + } + ) + await addSnapshotGlobalIndexes(k) + return users +} +async function observe(source, view, read, table, cursor) { + const physical = table === 'provenTxs' ? 'proven_txs' : 'proven_tx_reqs' + const key = table === 'provenTxs' ? 'provenTxId' : 'provenTxReqId' + const stats = async () => + await view.read(async trx => + Object.fromEntries( + ( + await source + .toDb(trx) + .raw( + 'SELECT OBJECT_NAME,COUNT_FETCH FROM performance_schema.table_io_waits_summary_by_table WHERE OBJECT_SCHEMA=DATABASE()' + ) + )[0].map(r => [r.OBJECT_NAME, Number(r.COUNT_FETCH)]) + ) + ) + const queries = [] + const listener = q => { + if (q.sql.startsWith('select') && q.sql.includes('snapshot_global_keys') && q.sql.includes('cross join')) + queries.push(q) + } + const before = await stats() + source.knex.on('query', listener) + let page + try { + page = await read( + table, + cursor === undefined ? undefined : { version: 1, snapshotId: 'global-native-seek', table, after: [cursor] }, + { maxRows: 16, maxBytes: 131072 } + ) + } finally { + source.knex.off('query', listener) + } + const after = await stats() + const fetches = Object.fromEntries(Object.keys(after).map(name => [name, after[name] - (before[name] ?? 0)])) + const expected = Array.from( + { length: Math.min(16, Math.ceil((8192 - (cursor ?? 0)) / 2)) }, + (_, i) => (cursor ?? 0) + i * 2 + 1 + ) + assert.deepEqual( + page.rows.map(row => row[key]), + expected + ) + assert.equal(fetches[physical], page.rows.length * 2, JSON.stringify(fetches)) + assert.equal(fetches.users ?? 0, 0) + assert.ok( + fetches.snapshot_global_keys >= page.rows.length * 2 && fetches.snapshot_global_keys <= page.rows.length * 2 + 4, + JSON.stringify(fetches) + ) + for (const [name, count] of Object.entries(fetches)) + if (![physical, 'snapshot_global_keys'].includes(name)) assert.equal(count, 0, name) + assert.equal(queries.length, 2) + const plans = [] + await runInSeries(queries, async q => { + const plan = await view.read(async trx => (await source.toDb(trx).raw('EXPLAIN ' + q.sql, q.bindings))[0]) + assert.equal(plan[0].table, 'snapshot_global_keys') + assert.equal(plan[0].key, 'snapshot_global_page') + assert.ok((cursor === undefined ? ['range', 'ref'] : ['range']).includes(plan[0].type), JSON.stringify(plan)) + plans.push(plan) + }) + return { table, cursor: cursor ?? null, rows: page.rows.length, fetches, plans } +} +async function qualifyMysqlGlobalIndexSeeks(control, connection) { + const database = 'ts569_global_read_' + randomUUID().replaceAll('-', '') + let source + try { + await control.raw('CREATE DATABASE ??', [database]) + source = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + }) + await source.migrate('synthetic global reader seek', 'synthetic-global-reader-seek') + await source.makeAvailable() + const [userId] = await seed(source), + results = [] + await runInSeries(['fresh', 'refreshed'], async statistics => { + if (statistics === 'refreshed') + await source.knex.raw('ANALYZE TABLE snapshot_global_keys,proven_txs,proven_tx_reqs') + const view = await source.openReadSnapshot() + try { + const read = pageReader(source, userId, 'global-native-seek', view, true, true, true, true) + await runInSeries(['provenTxs', 'provenTxReqs'], async table => { + await runInSeries([undefined, 1500, 7000, 8180], async cursor => { + results.push({ statistics, ...(await observe(source, view, read, table, cursor)) }) + }) + }) + } finally { + await view.close() + } + }) + return { actualReaderTwoPass: true, rows: 8192, results } + } finally { + if (source) await source.destroy() + await control.raw('DROP DATABASE IF EXISTS ??', [database]) + } +} +module.exports = { qualifyMysqlGlobalIndexSeeks } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotHistoricalMigrations.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotHistoricalMigrations.cjs new file mode 100644 index 000000000..bada5ecaf --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotHistoricalMigrations.cjs @@ -0,0 +1,16 @@ +// Native fixtures deliberately retain the schema whose migration they qualify. +const { KnexMigrations, SNAPSHOT_SQLITE_INDEX_MIGRATION } = require('../../out/src/storage/schema/KnexMigrations.js') +async function migrateBeforeSqliteGeneration(source, name, identity) { + const migrationSource = new KnexMigrations(source.chain, name, identity, 1024) + delete migrationSource.migrations[SNAPSHOT_SQLITE_INDEX_MIGRATION] + const config = { migrationSource, disableTransactions: false } + const sqlite = String(source.knex.client.config.client).includes('sqlite') + if (sqlite) await source.knex.raw('PRAGMA foreign_keys=OFF') + try { + await source.knex.migrate.latest(config) + return await source.knex.migrate.currentVersion(config) + } finally { + if (sqlite) await source.knex.raw('PRAGMA foreign_keys=ON') + } +} +module.exports = { migrateBeforeSqliteGeneration } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureMysql.cjs new file mode 100644 index 000000000..da7a4fe7f --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureMysql.cjs @@ -0,0 +1,222 @@ +const assert = require('node:assert/strict') +const { fork } = require('node:child_process') +const { mkdtemp, rm, open: openFile } = require('node:fs/promises') +const { tmpdir } = require('node:os') +const { join } = require('node:path') +const { + open, + StorageKnex, + StorageProvider, + seedArchiveClosure, + tables, + exact +} = require('./snapshotJournalMysqlConnection.cjs') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + installSnapshotJournalMysqlGeneration, + completeSnapshotJournalMysqlGeneration +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.js') +const { + copySnapshotJournalBootstrapPage +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const { readSnapshotJournalReceipt } = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') +const { snapshotArchiveTables } = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveStore.js') +const identity = '02' + '11'.repeat(32) +const request = { ceiling: '9223372036854775807', receiptPolicy: { receiptLimit: 128, receiptLifetimeMs: 600000 } } +async function killAt(phase, marker) { + const output = await openFile(marker, 'wx+', 0o600) + const child = fork(join(__dirname, 'snapshotJournalCaptureMysqlChild.cjs'), [phase], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc', output.fd], + env: process.env + }) + let stderr = '' + child.stderr.on('data', data => { + stderr += data + }) + const timer = setTimeout(() => child.kill('SIGKILL'), 20000) + try { + const result = await new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + assert.equal(result.signal, 'SIGKILL', stderr) + const bytes = Buffer.alloc(64) + const { bytesRead } = await output.read(bytes, 0, bytes.length, 0) + assert.equal(bytes.subarray(0, bytesRead).toString('utf8'), phase) + } finally { + clearTimeout(timer) + await output.close() + } +} +async function main() { + const k = open(), + peer = open(), + storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }), + results = [] + let failure + try { + await storage.migrate('capture native fixture', 'synthetic-capture-native') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identity), + { user: other } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(storage, user.userId, other.userId) + await installSnapshotJournalMysqlGeneration(k, request.ceiling, request.receiptPolicy) + let finished = false + function* pages() { + for (let n = 0; n < 100 && !finished; n++) yield n + } + await runInSeries(pages(), async () => { + finished = (await copySnapshotJournalBootstrapPage(k, 1000000)).complete + }) + assert(finished) + await completeSnapshotJournalMysqlGeneration(k, request.ceiling, request.receiptPolicy) + await exact(k) + await runInSeries(['READ COMMITTED', 'REPEATABLE READ'], async isolation => { + const foreground = open(), + provider = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: foreground }) + try { + await provider.makeAvailable() + await foreground.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + await peer.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + const started = performance.now(), + view = await provider.openSnapshotJournalSource(identity, request), + pinned = [] + try { + assert.deepEqual( + await peer.transaction(t => readSnapshotJournalReceipt(t, view.receiptBinding, view.receipt)), + view.receipt + ) + await runInSeries(snapshotArchiveTables, async table => { + const page = await view.readPage(table) + assert(page.done) + assert(page.rows.length > 0) + pinned.push(page.rows) + }) + const writing = performance.now() + await runInSeries(tables, async table => { + await peer(table).update({ + created_at: new Date(isolation === 'READ COMMITTED' ? '2026-01-02T00:00:00Z' : '2026-01-03T00:00:00Z') + }) + }) + assert(performance.now() - writing < 5000, 'Writer remained blocked by the retained view') + await runInSeries(snapshotArchiveTables.entries(), async ([index, table]) => { + const page = await view.readPage(table) + assert.deepEqual(page.rows, pinned[index]) + for (const row of page.rows) if ('userId' in row) assert.equal(row.userId, user.userId) + }) + assert.deepEqual( + pinned[0].map(row => row.provenTxId), + [1, 3] + ) + assert.deepEqual( + pinned[1].map(row => row.provenTxReqId), + [1, 3] + ) + assert.deepEqual( + pinned[12].map(row => row.syncStateId), + [1, 3] + ) + await assert.rejects(provider.openSnapshotArchiveSource(identity), /already has/) + const rows = await peer('snapshot_journal_physical').select(peer.raw('CAST(revision AS CHAR) revision')) + assert( + rows.some(row => BigInt(row.revision) > BigInt(view.receipt.highWater)), + JSON.stringify({ + isolation, + highWater: view.receipt.highWater, + maxRevision: rows.reduce((v, row) => (BigInt(row.revision) > BigInt(v) ? row.revision : v), '0') + }) + ) + } finally { + await view.close() + } + assert.equal(view.isOpen, false) + const held = await peer.transaction() + try { + await held('snapshot_journal_clock').where('id', 1).forUpdate().first() + const lockStart = performance.now() + await assert.rejects( + provider.openSnapshotJournalSource(identity, request), + error => error.code === 'ER_LOCK_NOWAIT' + ) + assert(performance.now() - lockStart < 2000) + } finally { + await held.rollback() + } + const next = await provider.openSnapshotJournalSource(identity, request) + assert.notEqual(next.receipt.requestId, view.receipt.requestId) + assert.equal(next.receiptBinding.backend, view.receiptBinding.backend) + assert(BigInt(next.receipt.highWater) > BigInt(view.receipt.highWater)) + await next.close() + const before = (await peer('information_schema.PROCESSLIST').where('DB', 'ts569_snapshot')).length + const last = await provider.openSnapshotJournalSource(identity, request) + const during = (await peer('information_schema.PROCESSLIST').where('DB', 'ts569_snapshot')).length + assert.equal(during, before + 2) + await provider.destroy() + assert.equal(last.isOpen, false) + await last.closed + await assert.rejects(last.readPage('txLabels')) + let after = (await peer('information_schema.PROCESSLIST').where('DB', 'ts569_snapshot')).length + const closeDeadline = performance.now() + 5000 + function* closing() { + while (after !== before - 1 && performance.now() < closeDeadline) yield undefined + } + await runInSeries(closing(), async () => { + await new Promise(resolve => setTimeout(resolve, 20)) + after = (await peer('information_schema.PROCESSLIST').where('DB', 'ts569_snapshot')).length + }) + assert.equal(after, before - 1, 'Server did not retire all provider-owned connections within five seconds') + // A fresh provider is used for the next isolation cohort; foreground pool + // ownership was deliberately drained by the preceding shutdown proof. + results.push({ + isolation, + tables: 13, + receiptCommitted: true, + immutable: true, + writerProgress: true, + lockRefusal: true, + nativeShutdown: true, + milliseconds: Math.round(performance.now() - started) + }) + } finally { + await provider.destroy() + } + }) + const directory = await mkdtemp(join(tmpdir(), 'ts569-capture-kill-')) + try { + await runInSeries(['before-commit', 'after-commit', 'opened'], async phase => { + const before = (await peer('snapshot_journal_receipts')).length + await killAt(phase, join(directory, phase)) + assert.equal((await peer('snapshot_journal_receipts')).length, before + Number(phase !== 'before-commit')) + const view = await storage.openSnapshotJournalSource(identity, request) + await assert.rejects(storage.openSnapshotJournalSource(identity, request), /already has/) + assert.deepEqual( + await peer.transaction(t => readSnapshotJournalReceipt(t, view.receiptBinding, view.receipt)), + view.receipt + ) + await view.close() + await peer('tx_labels') + .where('txLabelId', 1) + .update({ label: 'after process loss ' + phase }) + await exact(k) + results.push({ processLoss: phase, receiptAtomic: true, newCapture: true, writerProgress: true }) + }) + } finally { + await rm(directory, { recursive: true, force: true }) + } + console.log(JSON.stringify({ fixture: 'journal-capture-provider', results })) + } catch (error) { + failure = { error } + } + const closed = await Promise.allSettled([storage.destroy(), k.destroy(), peer.destroy()]) + const errors = closed.filter(result => result.status === 'rejected').map(result => result.reason) + if (errors.length) + throw new AggregateError( + [...(failure === undefined ? [] : [failure.error]), ...errors], + 'MySQL capture fixture cleanup failed' + ) + if (failure !== undefined) throw failure.error +} +main().catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureMysqlChild.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureMysqlChild.cjs new file mode 100644 index 000000000..e5d060c76 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureMysqlChild.cjs @@ -0,0 +1,23 @@ +const { open, StorageKnex, StorageProvider } = require('./snapshotJournalMysqlConnection.cjs') +const phase = process.argv[2] +const identity = '02' + '11'.repeat(32) +const request = { ceiling: '9223372036854775807', receiptPolicy: { receiptLimit: 128, receiptLifetimeMs: 600000 } } +const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: open() }) +const injection = require('./snapshotJournalCaptureProcessLoss.cjs')(phase) +process.once('disconnect', () => process.exit(1)) +const deadline = setTimeout(() => process.exit(2), 20000) +deadline.unref() +storage + .openSnapshotJournalSource(identity, request) + .then(async view => { + if (phase === 'opened') injection.park() + await view.close() + throw new Error('Did not reach capture boundary: ' + JSON.stringify({ phase, pools: injection.pools })) + }) + .catch(error => { + console.error(error) + process.exitCode = 1 + }) + .finally(async () => { + await storage.destroy() + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureProcessLoss.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureProcessLoss.cjs new file mode 100644 index 000000000..a391b51dd --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureProcessLoss.cjs @@ -0,0 +1,50 @@ +const assert = require('node:assert/strict') +const { writeFileSync } = require('node:fs') + +// The parent supplies an exclusive marker descriptor at stdio[4]; children +// cannot choose a filesystem write path through command-line arguments. +// Instrument only pools constructed after the foreground provider in this +// disposable child. SQL hooks never enter production code. +module.exports = function captureProcessLoss(phase) { + assert(['before-commit', 'after-commit', 'opened'].includes(phase)) + const native = require('knex'), + original = native.knex + let pools = 0 + const park = () => { + writeFileSync(4, phase) + process.kill(process.pid, 'SIGKILL') + } + native.knex = (...args) => { + pools++ + const owned = original(...args) + let receiptWritten = false + owned.on('query', q => { + if (q.sql.startsWith('insert into `snapshot_journal_receipts`')) receiptWritten = true + if (receiptWritten && q.sql === 'COMMIT;' && phase === 'before-commit') park() + }) + // Internal transaction COMMIT bypasses query-response. Await both SQL and + // executionPromise before terminating, while the provider still awaits its + // commit call and cannot publish the source. + const transaction = owned.client.transaction + owned.client.transaction = function (...parameters) { + const trx = transaction.apply(this, parameters) + const commit = trx.commit + trx.commit = async function (...values) { + const result = await commit.apply(this, values) + if (receiptWritten && phase === 'after-commit') { + await this.transactor.executionPromise + park() + } + return result + } + return trx + } + return owned + } + return { + park, + get pools() { + return pools + } + } +} diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureSqlite.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureSqlite.cjs new file mode 100644 index 000000000..fa88afe90 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalCaptureSqlite.cjs @@ -0,0 +1,170 @@ +const assert = require('node:assert/strict') +const { fork } = require('node:child_process') +const { mkdtemp, rm, open: openFile } = require('node:fs/promises') +const { tmpdir } = require('node:os') +const { join } = require('node:path') +const { + knex, + StorageKnex, + StorageProvider, + seedArchiveClosure, + tables, + exact +} = require('./snapshotJournalNativeFixture.cjs') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + installSnapshotJournalSqliteGeneration, + completeSnapshotJournalSqliteGeneration +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.js') +const { + copySnapshotJournalBootstrapPage +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const { readSnapshotJournalReceipt } = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') +const { snapshotArchiveTables } = require('../../out/src/storage/snapshot/archive/KnexSnapshotArchiveStore.js') +const identity = '02' + '11'.repeat(32) +const request = { ceiling: '9223372036854775807', receiptPolicy: { receiptLimit: 128, receiptLifetimeMs: 600000 } } +const open = filename => + knex({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true, pool: { min: 1, max: 1 } }) +async function child(filename, phase) { + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: open(filename) }) + const injection = require('./snapshotJournalCaptureProcessLoss.cjs')(phase) + process.once('disconnect', () => process.exit(1)) + const deadline = setTimeout(() => process.exit(2), 20000) + deadline.unref() + try { + const view = await storage.openSnapshotJournalSource(identity, request) + if (phase === 'opened') injection.park() + await view.close() + throw new Error('Did not reach capture boundary: ' + JSON.stringify({ phase, pools: injection.pools })) + } finally { + await storage.destroy() + } +} +async function killAt(filename, phase, marker) { + const output = await openFile(marker, 'wx+', 0o600) + const killed = fork(__filename, ['child', filename, phase], { stdio: ['ignore', 'ignore', 'pipe', 'ipc', output.fd] }) + let stderr = '' + killed.stderr.on('data', data => { + stderr += data + }) + const timer = setTimeout(() => killed.kill('SIGKILL'), 20000) + try { + const result = await new Promise((resolve, reject) => { + killed.once('error', reject) + killed.once('exit', (code, signal) => resolve({ code, signal })) + }) + assert.equal(result.signal, 'SIGKILL', stderr) + const bytes = Buffer.alloc(64) + const { bytesRead } = await output.read(bytes, 0, bytes.length, 0) + assert.equal(bytes.subarray(0, bytesRead).toString('utf8'), phase) + } finally { + clearTimeout(timer) + await output.close() + } +} +async function main() { + const directory = await mkdtemp(join(tmpdir(), 'ts569-capture-sqlite-')) + const filename = join(directory, 'wallet.sqlite'), + k = open(filename), + peer = open(filename) + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + const results = [] + let failure + try { + await k.raw('PRAGMA journal_mode=WAL') + await storage.migrate('capture native WAL fixture', 'synthetic-capture-native') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identity), + { user: other } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(storage, user.userId, other.userId) + await installSnapshotJournalSqliteGeneration(k, request.ceiling, request.receiptPolicy) + let finished = false + function* pages() { + for (let n = 0; n < 100 && !finished; n++) yield n + } + await runInSeries(pages(), async () => { + finished = (await copySnapshotJournalBootstrapPage(k, 1000000)).complete + }) + assert(finished) + await completeSnapshotJournalSqliteGeneration(k, request.receiptPolicy) + await exact(k) + const view = await storage.openSnapshotJournalSource(identity, request), + pinned = [] + try { + assert.deepEqual( + await peer.transaction(t => readSnapshotJournalReceipt(t, view.receiptBinding, view.receipt)), + view.receipt + ) + await runInSeries(snapshotArchiveTables, async table => { + const page = await view.readPage(table) + assert(page.done) + assert(page.rows.length > 0) + pinned.push(page.rows) + }) + const started = performance.now() + await runInSeries(tables, async table => { + await peer(table).update({ created_at: '2026-01-02T00:00:00.000Z' }) + }) + assert(performance.now() - started < 5000, 'WAL writer remained blocked by a retained reader') + await runInSeries(snapshotArchiveTables.entries(), async ([index, table]) => { + assert.deepEqual((await view.readPage(table)).rows, pinned[index]) + }) + assert.deepEqual( + pinned[0].map(row => row.provenTxId), + [1, 3] + ) + assert.deepEqual( + pinned[1].map(row => row.provenTxReqId), + [1, 3] + ) + assert.deepEqual( + pinned[12].map(row => row.syncStateId), + [1, 3] + ) + results.push({ tables: 13, receiptCommitted: true, immutable: true, writerProgress: true }) + } finally { + await view.close() + } + await runInSeries(['before-commit', 'after-commit', 'opened'], async phase => { + const before = (await peer('snapshot_journal_receipts')).length + await killAt(filename, phase, join(directory, phase)) + assert.equal((await peer('snapshot_journal_receipts')).length, before + Number(phase !== 'before-commit')) + const next = await storage.openSnapshotJournalSource(identity, request) + assert.deepEqual( + await peer.transaction(t => readSnapshotJournalReceipt(t, next.receiptBinding, next.receipt)), + next.receipt + ) + await next.close() + await peer('tx_labels') + .where('txLabelId', 1) + .update({ label: 'after process loss ' + phase }) + await exact(k) + assert.deepEqual(await k.raw('PRAGMA foreign_key_check'), []) + results.push({ processLoss: phase, receiptAtomic: true, newCapture: true, writerProgress: true }) + }) + console.log(JSON.stringify({ fixture: 'journal-capture-WAL', results })) + } catch (error) { + failure = { error } + } + const closed = await Promise.allSettled([storage.destroy(), k.destroy(), peer.destroy()]) + const errors = closed.filter(result => result.status === 'rejected').map(result => result.reason) + try { + await rm(directory, { recursive: true, force: true }) + } catch (error) { + errors.push(error) + } + if (errors.length) + throw new AggregateError( + [...(failure === undefined ? [] : [failure.error]), ...errors], + 'WAL capture fixture cleanup failed' + ) + if (failure !== undefined) throw failure.error +} +module.exports = main +if (require.main?.filename === __filename) { + const run = process.argv[2] === 'child' ? child(...process.argv.slice(3)) : main() + run.catch(error => { + console.error(error) + process.exitCode = 1 + }) +} diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceChild.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceChild.cjs new file mode 100644 index 000000000..0a3fc0c7b --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceChild.cjs @@ -0,0 +1,56 @@ +const assert = require('node:assert/strict') +const input = JSON.parse(process.argv[2]) +const { maintainSnapshotJournal } = require('../../out/src/storage/snapshot/journal/SnapshotJournalMaintenance.js') +const backend = require('../../out/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.js') +const { inject } = require('./snapshotJournalMaintenanceProcessLoss.cjs') +async function main() { + assert(['sqlite', 'mysql'].includes(input.backend)) + const k = + input.backend === 'mysql' + ? require('./snapshotJournalMysqlConnection.cjs').open(true) + : require('knex').knex({ + client: 'better-sqlite3', + connection: { filename: input.filename }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + }) + process.once('disconnect', () => process.exit(1)) + const deadline = setTimeout(() => process.exit(2), 20000) + deadline.unref() + const bind = backend.bindSnapshotJournalCaptureBackend + backend.bindSnapshotJournalCaptureBackend = async (...args) => { + const value = await bind(...args) + inject(args[0], input.operation, input.phase) + return value + } + try { + if (input.backend === 'mysql') { + assert(['READ COMMITTED', 'REPEATABLE READ'].includes(input.isolation)) + k.client.config.pool = { + ...k.client.config.pool, + afterCreate(connection, done) { + connection.query('SET SESSION TRANSACTION ISOLATION LEVEL ' + input.isolation, error => + done(error, connection) + ) + } + } + } + const operation = + input.operation === 'floor' ? { kind: 'floor', floor: input.floor } : { kind: 'collect', page: input.request } + const task = maintainSnapshotJournal(async () => k.client.config, { + epoch: input.epoch, + ceiling: input.ceiling, + receiptPolicy: input.receiptPolicy, + operation + }) + await task.result + await task.closed + throw new Error('Did not reach owned maintenance process-loss boundary') + } finally { + await k.destroy() + } +} +main().catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceCuts.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceCuts.cjs new file mode 100644 index 000000000..286704304 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceCuts.cjs @@ -0,0 +1,140 @@ +const { maintainSnapshotJournal } = require('../../out/src/storage/snapshot/journal/SnapshotJournalMaintenance.js') +async function maintain(k, input, operation) { + const task = maintainSnapshotJournal(async () => k.client.config, { + epoch: input.epoch, + ceiling: input.ceiling, + receiptPolicy: input.receiptPolicy, + operation + }) + const result = await task.result + await task.closed + return result.value +} +const assert = require('node:assert/strict') +const { join } = require('node:path') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { tables, exact } = require('./snapshotJournalNativeFixture.cjs') +const { killAt } = require('./snapshotJournalMaintenanceProcessLoss.cjs') +const { collectSnapshotJournalReceipts } = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') + +const { snapshotArchiveDatabaseNow } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveSql.js') +const { snapshotJournalRevisionText } = require('../../out/src/storage/snapshot/journal/SnapshotJournalRevisionSql.js') +async function source(k) { + const rows = {} + await runInSeries(tables, async table => { + rows[table] = await k(table) + }) + return rows +} +async function newest(k) { + let prefix = '0' + await runInSeries(['scope', 'physical'], async stream => { + const table = 'snapshot_journal_' + stream + const row = await k(table) + .select({ revisionText: snapshotJournalRevisionText(k, 'revision') }) + .orderBy(table + '.revision', 'desc') + .first() + if (row !== undefined && BigInt(row.revisionText) > BigInt(prefix)) prefix = row.revisionText + }) + assert(BigInt(prefix) > 0n) + return prefix +} +module.exports = async function cuts(k, input) { + await k('snapshot_journal_receipts').update({ expiresAt: (await snapshotArchiveDatabaseNow(k)) - 1 }) + await k.transaction(t => collectSnapshotJournalReceipts(t)) + assert.equal((await k('snapshot_journal_receipts')).length, 0) + const results = [] + const clock = async () => + ( + await k('snapshot_journal_clock') + .select({ revision: snapshotJournalRevisionText(k, 'revision') }) + .first() + ).revision + await runInSeries(['before-commit', 'after-commit'], async phase => { + await k('tx_labels') + .where('txLabelId', 1) + .update({ label: 'floor process loss ' + phase }) + const floor = await newest(k), + previous = String((await k('snapshot_journal_retention').first()).floor), + beforeClock = input.backend === 'sqlite' ? await clock() : undefined + const beforeSource = await source(k) + await killAt({ ...input, operation: 'floor', phase, floor, marker: join(input.directory, 'floor-' + phase) }) + assert.equal( + String((await k('snapshot_journal_retention').first()).floor), + phase === 'after-commit' ? floor : previous + ) + if (input.backend === 'sqlite') + assert.equal(await clock(), phase === 'after-commit' ? String(BigInt(beforeClock) + 1n) : beforeClock) + assert.deepEqual(await source(k), beforeSource) + const recovered = await maintain(k, input, { kind: 'floor', floor }) + assert.equal(recovered.floor, floor) + await exact(k) + results.push({ operation: 'floor', phase, atomic: true, lostAcknowledgementRecovered: phase === 'after-commit' }) + }) + let firstId = 1000 + await runInSeries(['scope', 'physical'], async stream => { + await runInSeries(['after-first-delete', 'before-commit', 'after-commit'], async phase => { + const ids = [firstId++, firstId++, firstId++], + date = input.backend === 'mysql' ? new Date('2026-01-01T00:00:00Z') : '2026-01-01T00:00:00.000Z' + await k('tx_labels').insert( + ids.map(id => ({ + txLabelId: id, + userId: input.userId, + label: 'gc process loss ' + id, + isDeleted: false, + created_at: date, + updated_at: date + })) + ) + await k('tx_labels').whereIn('txLabelId', ids).delete() + const floor = await newest(k) + await maintain(k, input, { kind: 'floor', floor }) + const key = { + tableId: 3, + ...(stream === 'scope' ? { userId: input.userId } : {}), + id1: ids[0] - 1, + id2: 0, + exactText: '' + } + const request = { epoch: input.epoch, floor, stream, limit: 3, after: { epoch: input.epoch, floor, stream, key } } + const table = 'snapshot_journal_' + stream, + order = + stream === 'scope' ? ['tableId', 'userId', 'id1', 'id2', 'exactText'] : ['tableId', 'id1', 'id2', 'exactText'] + const before = await k(table).orderBy(order), + beforeSource = await source(k), + beforeClock = input.backend === 'sqlite' ? await clock() : undefined + const deleted = row => + Number(row.tableId) === 3 && + ids.includes(Number(row.id1)) && + (stream !== 'scope' || Number(row.userId) === input.userId) + assert.equal(before.filter(deleted).length, 3) + assert(before.filter(deleted).every(row => Number(row.present) === 0)) + await killAt({ ...input, operation: stream, phase, request, marker: join(input.directory, stream + '-' + phase) }) + assert.deepEqual( + await k(table).orderBy(order), + phase === 'after-commit' ? before.filter(row => !deleted(row)) : before + ) + if (input.backend === 'sqlite') + assert.equal(await clock(), phase === 'after-commit' ? String(BigInt(beforeClock) + 1n) : beforeClock) + assert.deepEqual(await source(k), beforeSource) + const recovered = await maintain(k, input, { kind: 'collect', page: request }) + assert.equal(recovered.removed, phase === 'after-commit' ? 0 : 3) + assert(recovered.examined <= 3) + await exact(k) + assert.deepEqual(await source(k), beforeSource) + await k('tx_labels') + .where('txLabelId', 1) + .update({ label: 'writer after ' + stream + ' ' + phase }) + await exact(k) + results.push({ + operation: stream, + phase, + atomic: true, + sourcePreserved: true, + recovered: true, + writerProgress: true + }) + }) + }) + return results +} diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceFixture.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceFixture.cjs new file mode 100644 index 000000000..cf09456b5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceFixture.cjs @@ -0,0 +1,301 @@ +const assert = require('node:assert/strict') +const { mkdtemp, rm } = require('node:fs/promises') +const { tmpdir } = require('node:os') +const { join } = require('node:path') +const { + maintainSnapshotJournal: maintain +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalMaintenance.js') +const native = require('./snapshotJournalNativeFixture.cjs') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + copySnapshotJournalBootstrapPage +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const sqliteGeneration = require('../../out/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.js') +const mysqlGeneration = require('../../out/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.js') +const backend = require('../../out/src/storage/snapshot/journal/SnapshotJournalCaptureBackend.js') +const identity = '02' + '11'.repeat(32) +const policy = { receiptLimit: 128, receiptLifetimeMs: 600000 }, + ceiling = '9223372036854775807' +function gate() { + let resolve + const promise = new Promise(yes => { + resolve = yes + }) + return { promise, resolve } +} +module.exports = async function proof(isolation) { + const local = isolation === 'WAL', + directory = await mkdtemp(join(tmpdir(), 'ts569-maintenance-controller-')) + const mysql = local ? undefined : require('./snapshotJournalMysqlConnection.cjs') + const config = { + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 0, max: 1 } + } + const k = local ? native.knex(config) : mysql.open(true) + const peer = local ? native.knex(config) : mysql.open(true) + const { StorageKnex: Provider } = require('../../out/src/storage/StorageKnex.js') + const storage = new Provider({ ...native.StorageProvider.createStorageBaseOptions('test'), knex: k }) + const results = [], + pools = [], + originalClose = backend.closeSnapshotJournalCapturePool, + originalBind = backend.bindSnapshotJournalCaptureBackend + backend.closeSnapshotJournalCapturePool = async (...args) => { + await originalClose(...args) + pools.push(args[1]) + } + const request = (epoch, operation) => ({ epoch, ceiling, receiptPolicy: policy, operation }) + const run = async input => { + const value = await storage.maintainSnapshotJournal(input) + await storage.awaitSnapshotJournalMaintenanceCleanup() + return value + } + try { + if (local) { + await k.raw('PRAGMA journal_mode=WAL') + await k.raw('PRAGMA busy_timeout=0') + await peer.raw('PRAGMA busy_timeout=0') + } else { + await k.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + await peer.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + k.client.config.pool = { + ...k.client.config.pool, + afterCreate(connection, done) { + connection.query('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation, error => done(error, connection)) + } + } + backend.bindSnapshotJournalCaptureBackend = async (...args) => { + const value = await originalBind(...args) + const [[write]] = await args[0].raw('SELECT @@transaction_isolation isolationLevel').connection(args[1]) + const [[read]] = await args[2].raw('SELECT @@transaction_isolation isolationLevel').connection(args[3]) + assert.equal(write.isolationLevel, isolation.replaceAll(' ', '-')) + assert.equal(read.isolationLevel, isolation.replaceAll(' ', '-')) + return value + } + } + await storage.migrate('owned bounded journal maintenance', 'synthetic-maintenance-controller') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identity), + { user: other } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await native.seedArchiveClosure(storage, user.userId, other.userId) + if (local) await sqliteGeneration.installSnapshotJournalSqliteGeneration(k, ceiling, policy) + else await mysqlGeneration.installSnapshotJournalMysqlGeneration(k, ceiling, policy) + let done = false + function* pages() { + for (let n = 0; n < 100 && !done; n++) yield n + } + await runInSeries(pages(), async () => { + done = (await copySnapshotJournalBootstrapPage(k, 1000000)).complete + }) + assert(done) + const generation = local + ? await sqliteGeneration.completeSnapshotJournalSqliteGeneration(k, policy) + : await mysqlGeneration.completeSnapshotJournalMysqlGeneration(k, ceiling, policy) + const epoch = generation.epoch + const date = local ? '2026-01-01T00:00:00.000Z' : new Date('2026-01-01T00:00:00.000Z') + await k('tx_labels').insert({ + txLabelId: 999, + userId: user.userId, + label: 'collectible', + isDeleted: false, + created_at: date, + updated_at: date + }) + await k('tx_labels').where('txLabelId', 999).delete() + const revisionSql = + 'SELECT CAST(MAX(revision) AS ' + + (local ? 'TEXT' : 'CHAR') + + ') revision FROM (SELECT revision FROM snapshot_journal_scope UNION ALL SELECT revision FROM snapshot_journal_physical) AS revisions' + const [[row]] = local ? [await k.raw(revisionSql)] : await k.raw(revisionSql) + await native.exact(k) + const arrays = async () => { + const rows = [] + await runInSeries(native.tables, async table => { + rows.push((await k(table)).sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b)))) + }) + return rows + } + const before = await arrays() + const floor = await run(request(epoch, { kind: 'floor', floor: row.revision })) + assert.equal(floor.kind, 'floor') + assert.equal(floor.value.floor, row.revision) + results.push('owned complete generation advances floor and physically closes both pools') + await runInSeries(['scope', 'physical'], async stream => { + let after, + finished = false, + examined = 0, + removed = 0 + function* pages() { + for (let n = 0; n < 100 && !finished; n++) yield n + } + await runInSeries(pages(), async () => { + const answer = await run( + request(epoch, { kind: 'collect', page: { epoch, floor: row.revision, stream, limit: 4, after } }) + ) + assert.equal(answer.kind, 'collect') + assert(answer.value.examined <= 4) + examined += answer.value.examined + removed += answer.value.removed + after = answer.value.after + finished = answer.value.complete + }) + assert(finished) + assert(examined > 4) + assert.equal(removed, 1) + }) + await native.exact(k) + assert.deepEqual(await arrays(), before) + results.push('bounded multi-page collection preserves all thirteen source arrays') + const stale = request('00000000-0000-4000-8000-000000000000', { kind: 'floor', floor: row.revision }) + const rejected = maintain(async () => k.client.config, stale) + await assert.rejects(rejected.result, /generation/) + await assert.rejects(rejected.closed, /generation/) + assert.deepEqual(await arrays(), before) + results.push('changed generation refuses without source mutation') + const lock = await peer.transaction() + try { + await lock('knex_migrations_lock').where('index', 1).update({ is_locked: 1 }) + const rejected = maintain(async () => k.client.config, request(epoch, { kind: 'floor', floor: row.revision })) + const start = performance.now() + await assert.rejects(rejected.result) + await assert.rejects(rejected.closed) + assert(performance.now() - start < 2000) + } finally { + await lock.rollback() + } + results.push('actual held migration owner refuses boundedly') + if (!local) { + const entered = gate(), + release = gate(), + read = mysqlGeneration.readSnapshotJournalMysqlGeneration + mysqlGeneration.readSnapshotJournalMysqlGeneration = async (...args) => { + entered.resolve() + await release.promise + return await read(...args) + } + const task = maintain(async () => k.client.config, request(epoch, { kind: 'floor', floor: row.revision })) + try { + await entered.promise + const start = performance.now() + await peer('tx_labels').where('txLabelId', 1).update({ label: 'foreground during generation verification' }) + assert(performance.now() - start < 2000) + release.resolve() + await task.result + await task.closed + results.push('foreground wallet write proceeds during actual controller generation validation') + } finally { + release.resolve() + mysqlGeneration.readSnapshotJournalMysqlGeneration = read + await task.closed.catch(() => {}) + } + } + assert(pools.length >= 6) + assert(pools.every(connection => (local ? connection.open === false : connection.stream.closed))) + const cuts = await require('./snapshotJournalMaintenanceCuts.cjs')(k, { + backend: local ? 'sqlite' : 'mysql', + filename: config.connection.filename, + isolation, + directory, + userId: user.userId, + epoch, + ceiling, + receiptPolicy: policy + }) + assert.equal(cuts.length, 8) + results.push({ processLoss: cuts }) + row.revision = String((await k('snapshot_journal_retention').first()).floor) + const entered = gate(), + release = gate(), + controller = new AbortController() + const receipts = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js'), + advance = receipts.advanceSnapshotJournalFloor + const prior = await k('snapshot_journal_retention').first() + receipts.advanceSnapshotJournalFloor = async (...args) => { + const value = await advance(...args) + entered.resolve() + await release.promise + return value + } + try { + const pending = storage.maintainSnapshotJournal(request(epoch, { kind: 'floor', floor: row.revision }), { + signal: controller.signal + }) + const rejection = assert.rejects(pending, /cancelled/) + await entered.promise + controller.abort() + await rejection + await assert.rejects( + storage.maintainSnapshotJournal(request(epoch, { kind: 'floor', floor: row.revision })), + /already/ + ) + await assert.rejects(storage.openSnapshotJournalSource(identity, { ceiling, receiptPolicy: policy }), /already/) + let closed = false + const cleanup = storage.awaitSnapshotJournalMaintenanceCleanup().then(() => { + closed = true + }) + await new Promise(resolve => setImmediate(resolve)) + assert.equal(closed, false) + release.resolve() + await cleanup + assert.equal((await k('snapshot_journal_retention').first()).floor, prior.floor) + results.push('provider cancellation retains shared source admission until rollback and physical cleanup') + } finally { + release.resolve() + receipts.advanceSnapshotJournalFloor = advance + } + await run(request(epoch, { kind: 'floor', floor: row.revision })) + const resolve = storage.concurrentSnapshotReaderConfig.bind(storage), + enteredConfig = gate(), + releaseConfig = gate(), + controller2 = new AbortController() + storage.concurrentSnapshotReaderConfig = async () => { + enteredConfig.resolve() + await releaseConfig.promise + return await resolve() + } + const pending = storage.maintainSnapshotJournal(request(epoch, { kind: 'floor', floor: row.revision }), { + signal: controller2.signal + }) + const rejection = assert.rejects(pending) + await enteredConfig.promise + let destroyed = false + const destroying = storage.destroy().then(() => { + destroyed = true + }) + await rejection + await new Promise(resolve => setImmediate(resolve)) + assert.equal(destroyed, false) + await assert.rejects( + storage.maintainSnapshotJournal(request(epoch, { kind: 'floor', floor: row.revision })), + /destruction/ + ) + releaseConfig.resolve() + await destroying + results.push( + 'provider destruction fences admission synchronously and drains pending configuration without allocating' + ) + console.log( + JSON.stringify({ + fixture: 'owned bounded journal maintenance', + isolation, + results, + nativePoolsClosed: pools.length, + limitations: [ + 'no runtime quotas or registered generation lifecycle qualification', + 'component qualification does not establish full program or production readiness' + ] + }) + ) + } finally { + backend.closeSnapshotJournalCapturePool = originalClose + backend.bindSnapshotJournalCaptureBackend = originalBind + await Promise.allSettled([storage.destroy(), peer.destroy()]) + await rm(directory, { recursive: true, force: true }) + } +} +if (require.main?.filename === __filename) + module.exports('WAL').catch(error => { + console.error(error) + process.exitCode = 1 + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceMysqlRc.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceMysqlRc.cjs new file mode 100644 index 000000000..2be420c6a --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceMysqlRc.cjs @@ -0,0 +1,4 @@ +require('./snapshotJournalMaintenanceFixture.cjs')('READ COMMITTED').catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceMysqlRr.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceMysqlRr.cjs new file mode 100644 index 000000000..070ce4bbe --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceMysqlRr.cjs @@ -0,0 +1,4 @@ +require('./snapshotJournalMaintenanceFixture.cjs')('REPEATABLE READ').catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceProcessLoss.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceProcessLoss.cjs new file mode 100644 index 000000000..144b0bab1 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceProcessLoss.cjs @@ -0,0 +1,67 @@ +const assert = require('node:assert/strict') +const { fork } = require('node:child_process') +const { writeFileSync } = require('node:fs') +const { open } = require('node:fs/promises') +const { join } = require('node:path') + +// These hooks belong only to the disposable fixture's own native pool. +function inject(k, operation, phase) { + assert(['floor', 'scope', 'physical'].includes(operation)) + assert(['after-first-delete', 'before-commit', 'after-commit'].includes(phase)) + assert(operation !== 'floor' || phase !== 'after-first-delete') + const prefix = + operation === 'floor' ? 'update `snapshot_journal_retention`' : 'delete from `snapshot_journal_' + operation + '`' + let changed = false + const park = () => { + writeFileSync(4, phase) + process.kill(process.pid, 'SIGKILL') + } + k.on('query', q => { + if (q.sql.startsWith(prefix)) changed = true + if (changed && q.sql === 'COMMIT;' && phase === 'before-commit') park() + }) + k.on('query-response', (_response, q) => { + if (q.sql.startsWith(prefix) && phase === 'after-first-delete') park() + }) + const transaction = k.client.transaction + k.client.transaction = function (...parameters) { + const trx = transaction.apply(this, parameters) + const commit = trx.commit + trx.commit = async function (...values) { + const result = await commit.apply(this, values) + if (changed && phase === 'after-commit') { + await this.transactor.executionPromise + park() + } + return result + } + return trx + } +} +async function killAt(input) { + const { marker, ...childInput } = input + const output = await open(marker, 'wx+', 0o600) + const child = fork(join(__dirname, 'snapshotJournalMaintenanceChild.cjs'), [JSON.stringify(childInput)], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc', output.fd] + }) + let stderr = '' + child.stderr.on('data', data => { + stderr += data + }) + const timer = setTimeout(() => child.kill('SIGKILL'), 20000) + try { + const result = await new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + assert.equal(result.signal, 'SIGKILL', stderr) + const bytes = Buffer.alloc(64) + const { bytesRead } = await output.read(bytes, 0, bytes.length, 0) + assert.equal(bytes.subarray(0, bytesRead).toString('utf8'), input.phase) + } finally { + clearTimeout(timer) + await output.close() + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + } +} +module.exports = { inject, killAt } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceWal.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceWal.cjs new file mode 100644 index 000000000..a4e5fb86f --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMaintenanceWal.cjs @@ -0,0 +1,4 @@ +require('./snapshotJournalMaintenanceFixture.cjs')('WAL').catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysql.cjs new file mode 100644 index 000000000..bfc14053f --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysql.cjs @@ -0,0 +1,333 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const receiptPolicy = { receiptLimit: 128, receiptLifetimeMs: 2592000000 } +const assert = require('node:assert/strict'), + { fork } = require('node:child_process'), + { mkdtemp, rm } = require('node:fs/promises'), + { tmpdir } = require('node:os'), + { join } = require('node:path'), + { writeFileSync, readFileSync } = require('node:fs') +const { + open, + seedArchiveClosure, + StorageKnex, + StorageProvider, + exact, + tables +} = require('./snapshotJournalMysqlConnection.cjs') +const { + installSnapshotJournalMysqlGeneration: install, + readSnapshotJournalMysqlGeneration: read, + completeSnapshotJournalMysqlGeneration: complete +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.js') +const { + copySnapshotJournalBootstrapPage: copy +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const ceiling = '9223372036854775807', + intent = 'snapshot_journal_generation' +async function isolate(k, isolation) { + assert(['READ COMMITTED', 'REPEATABLE READ'].includes(isolation)) + await k.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + const [[level]] = await k.raw('SELECT @@transaction_isolation isolation') + assert.equal(level.isolation.replaceAll('-', ' '), isolation) + await k.raw('SET SESSION innodb_lock_wait_timeout=5') +} +async function bootstrap(k) { + let complete = false + function* pages() { + for (let i = 0; i < 100 && !complete; i++) yield i + } + await runInSeries(pages(), async () => { + const page = await copy(k, 1000000) + assert(!page.invalidated) + complete = page.complete + }) + assert(complete, 'Bootstrap did not complete within its bounded fixture pages') +} +async function finish(k) { + await bootstrap(k) + return await complete(k, ceiling, receiptPolicy) +} + +async function clear(k) { + const [triggers] = await k.raw( + "SELECT TRIGGER_NAME name FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND LEFT(TRIGGER_NAME,17)='snapshot_journal_'" + ) + await runInSeries(triggers, async row => { + await k.raw('DROP TRIGGER ??', [row.name]) + }) + const [names] = await k.raw( + "SELECT TABLE_NAME name FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LEFT(TABLE_NAME,17)='snapshot_journal_'" + ) + await runInSeries(names, async row => { + await k.schema.dropTable(row.name) + }) +} +async function rows(k) { + const result = {} + await runInSeries(tables, async name => { + result[name] = (await k(name).select('*')).map(row => JSON.stringify(row)).sort() + }) + return result +} +async function child() { + process.once('disconnect', () => process.exit(1)) + const childDeadline = setTimeout(() => process.exit(1), 20000) + childDeadline.unref() + const k = open(), + boundary = process.argv[3], + marker = process.argv[4], + isolation = process.argv[5] + let completing = false + const park = phase => { + if (boundary === phase) { + writeFileSync(marker, phase) + process.kill(process.pid, 'SIGKILL') + } + } + const object = sql => /^CREATE (?:TABLE|TRIGGER) (snapshot_journal_\w+)/.exec(sql)?.[1] + k.on('query', q => { + const name = object(q.sql) + if (name) park('before-' + name) + if (completing && q.sql === 'COMMIT;') park('complete-before-commit') + }) + k.on('query-response', (_r, q) => { + const name = object(q.sql) + if (name) park('after-' + name) + if (q.sql.startsWith('update `snapshot_journal_generation` set `nextObject`')) park('after-ack-' + q.bindings[0]) + if (completing && q.sql === 'COMMIT;') park('complete-after-commit') + if (q.sql.startsWith('update `snapshot_journal_bootstrap` set `rowLimit`')) park('bootstrap-after-budget-bind') + if (q.sql.startsWith('insert into `snapshot_journal_physical`')) park('bootstrap-after-metadata') + if (q.sql.startsWith('update `snapshot_journal_bootstrap` set `cursor`')) park('bootstrap-after-progress') + }) + try { + await isolate(k, isolation) + await install(k, ceiling, receiptPolicy) + if (boundary.startsWith('bootstrap-')) { + await copy(k, 1000000) + park('bootstrap-after-commit') + } + if (boundary.startsWith('complete-')) { + await bootstrap(k) + completing = true + await complete(k, ceiling, receiptPolicy) + park('complete-after-commit') + } + throw new Error('Crash boundary missed: ' + boundary) + } finally { + await k.destroy() + } +} +async function killAt(boundary, marker, isolation) { + const p = fork(__filename, ['child', boundary, marker, isolation], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'] + }) + let stderr = '' + p.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-6000) + }) + const timer = setTimeout(() => p.kill('SIGKILL'), 20000), + result = await new Promise((resolve, reject) => { + p.once('error', reject) + p.once('exit', (code, signal) => resolve({ code, signal })) + }) + clearTimeout(timer) + assert.equal(result.signal, 'SIGKILL', stderr) + assert.equal(readFileSync(marker, 'utf8'), boundary) +} +async function main() { + const k = open(), + source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }), + directory = await mkdtemp(join(tmpdir(), 'ts569-mysql-generation-')), + results = [] + try { + await source.migrate('journal generation fixture', 'synthetic-journal-generation') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)), + { user: foreign } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, foreign.userId) + const baseline = await rows(k) + await runInSeries(['READ COMMITTED', 'REPEATABLE READ'], async isolation => { + await isolate(k, isolation) + await clear(k) + const created = await install(k, ceiling, receiptPolicy) + assert.equal(created.nextObject, 59) + assert.equal(created.complete, false) + assert.equal(created.enabled, true) + assert.deepEqual(await install(k, ceiling, receiptPolicy), created) + assert.deepEqual(await read(k, ceiling, receiptPolicy), created) + await assert.rejects(complete(k, ceiling, receiptPolicy), /Invalid or unowned/) + const outer = await k.transaction(), + independent = open() + try { + await isolate(independent, isolation) + await outer('snapshot_journal_bootstrap').first() + assert.equal((await copy(independent, 1000000)).invalidated, false) + const progress = await independent('snapshot_journal_bootstrap').first() + await assert.rejects(copy(outer, 1000000), /Invalid snapshot journal bootstrap/) + assert.deepEqual(await independent('snapshot_journal_bootstrap').first(), progress) + } finally { + await outer.rollback() + await independent.destroy() + } + const completed = await finish(k) + await exact(k) + assert.equal(completed.complete, true) + assert.equal(completed.epoch, created.epoch) + assert.deepEqual(await read(k, ceiling, receiptPolicy), completed) + assert.deepEqual(await complete(k, ceiling, receiptPolicy), completed) + await k.transaction(async t => assert.deepEqual(await read(t, ceiling, receiptPolicy), completed)) + await runInSeries( + [ + [ + 'owner-comment', + "ALTER TABLE snapshot_journal_clock COMMENT='foreign'", + "ALTER TABLE snapshot_journal_clock COMMENT='snapshot-journal-owner:" + created.epoch + "'" + ], + [ + 'index', + 'CREATE INDEX foreign_generation_index ON snapshot_journal_physical(present)', + 'DROP INDEX foreign_generation_index ON snapshot_journal_physical' + ], + [ + 'check', + 'ALTER TABLE snapshot_journal_clock ALTER CHECK snapshot_journal_clock_chk_1 NOT ENFORCED', + 'ALTER TABLE snapshot_journal_clock ALTER CHECK snapshot_journal_clock_chk_1 ENFORCED' + ], + [ + 'foreign-observer', + 'CREATE TRIGGER foreign_generation_observer AFTER UPDATE ON snapshot_journal_clock FOR EACH ROW BEGIN DO 0; END', + 'DROP TRIGGER foreign_generation_observer' + ], + [ + 'source-binding', + "ALTER TABLE tx_labels ALTER label SET DEFAULT 'changed'", + 'ALTER TABLE tx_labels ALTER label DROP DEFAULT' + ] + ], + async ([, up, down]) => { + await k.raw(up) + await assert.rejects(read(k, ceiling, receiptPolicy), /Invalid or unowned/) + await assert.rejects(install(k, ceiling, receiptPolicy), /Invalid or unowned/) + await k.raw(down) + assert.deepEqual(await read(k, ceiling, receiptPolicy), completed) + } + ) + await k(intent).update({ nextObject: 58, complete: 0 }) + await k.raw('CREATE TABLE snapshot_journal_foreign(id INT)') + await assert.rejects(install(k, ceiling, receiptPolicy), /Invalid or unowned/) + assert(await k.schema.hasTable('snapshot_journal_foreign')) + await k.schema.dropTable('snapshot_journal_foreign') + await k(intent).update({ nextObject: 59, complete: 1 }) + await k('snapshot_journal_clock').delete() + await assert.rejects(install(k, ceiling, receiptPolicy), /Invalid or unowned/) + await k('snapshot_journal_clock').insert({ id: 1, ceiling }) + await k('snapshot_journal_bootstrap').update({ stream: 16, cursor: null }) + await assert.rejects(read(k, ceiling, receiptPolicy), /Invalid or unowned/) + await k('snapshot_journal_bootstrap').update({ stream: 17, cursor: null }) + await exact(k) + assert.deepEqual(await rows(k), baseline) + await clear(k) + const boundaries = [ + 'before-snapshot_journal_generation', + 'after-snapshot_journal_generation', + 'before-snapshot_journal_clock', + 'after-snapshot_journal_clock', + 'after-snapshot_journal_bootstrap', + 'before-snapshot_journal_retention', + 'after-snapshot_journal_retention', + 'before-snapshot_journal_receipts', + 'after-snapshot_journal_receipts', + 'after-snapshot_journal_scope_0_INSERT', + 'after-ack-7', + 'after-snapshot_journal_physical_12_DELETE', + 'after-ack-59', + 'bootstrap-after-budget-bind', + 'bootstrap-after-metadata', + 'bootstrap-after-progress', + 'bootstrap-after-commit', + 'complete-before-commit', + 'complete-after-commit' + ] + await runInSeries(boundaries, async boundary => { + await killAt(boundary, join(directory, isolation.replaceAll(' ', '-') + '-' + boundary), isolation) + const saved = (await k.schema.hasTable(intent)) ? await k(intent).first() : undefined + if (boundary.startsWith('complete-')) assert.equal(saved.complete, boundary === 'complete-after-commit' ? 1 : 0) + if (boundary.startsWith('bootstrap-')) { + const committed = boundary === 'bootstrap-after-commit', + progress = await k('snapshot_journal_bootstrap').first() + assert.equal(progress.rowsUsed, committed ? baseline.transactions.length : 0) + assert.equal(progress.rowLimit, committed ? 1000000 : null) + assert.equal( + progress.cursor, + committed + ? JSON.stringify([Math.max(...baseline.transactions.map(text => JSON.parse(text).transactionId))]) + : null + ) + } + const resumed = await install(k, ceiling, receiptPolicy) + if (saved) assert.equal(resumed.epoch, saved.epoch) + await finish(k) + let charged = 0 + await runInSeries( + [ + ...tables, + 'snapshot_profile_keys', + 'snapshot_relation_keys', + 'snapshot_certificate_field_keys', + 'snapshot_global_keys' + ], + async table => { + charged += Number((await k(table).count('* AS n').first()).n) + } + ) + assert.equal((await k('snapshot_journal_bootstrap').first()).rowsUsed, charged) + await exact(k) + assert.deepEqual(await rows(k), baseline) + await k('tx_labels').where('txLabelId', 1).update({ label: 'post-recovery writer' }) + await k('tx_labels') + .where('txLabelId', 1) + .update({ + label: JSON.parse(baseline.tx_labels.find(row => JSON.parse(row).txLabelId === 1)).label + }) + await exact(k) + assert.deepEqual(await rows(k), baseline) + await clear(k) + console.log(JSON.stringify({ isolation, boundary, status: 'passed' })) + }) + results.push({ + isolation, + installedObjects: 60, + exactResumeAndSourcePreserved: true, + bootstrapAndObservers: true, + refusals: true, + clientSigkillBoundaries: boundaries.length, + completionAtomic: true, + writerBarrierReleased: true + }) + }) + console.log( + JSON.stringify({ + status: 'MySQL journal generation', + results, + readerAdvertised: false, + limitations: [ + 'client SIGKILL only; server-crash/replication/failover remains open', + 'this generation fixture does not qualify capture; registered migration, floor/quota and receiver remain incomplete' + ] + }) + ) + } finally { + await source.destroy() + await rm(directory, { recursive: true, force: true }) + } +} +if (process.argv[2] === 'child') + child().catch(e => { + console.error(e) + process.exitCode = 1 + }) +else + main().catch(e => { + console.error(e) + process.exitCode = 1 + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysqlConnection.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysqlConnection.cjs new file mode 100644 index 000000000..1d05ee60f --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysqlConnection.cjs @@ -0,0 +1,34 @@ +const { execFileSync } = require('node:child_process') +const { knex } = require('knex') +const { executable, context, validateContext, validateContainer } = require('./snapshotArchiveDocker.cjs') +const container = process.env.TS_STACK_SNAPSHOT_CONTAINER +const expectedId = process.env.TS_STACK_SNAPSHOT_CONTAINER_ID +const owner = process.env.TS_STACK_SNAPSHOT_CONTAINER_OWNER +const secret = process.env.TS_STACK_SNAPSHOT_MYSQL_SECRET +if (!container || !expectedId || !owner || !secret) throw new Error('Use the bounded fixture launcher') +const docker = (...args) => + execFileSync(executable, ['--context', context, ...args], { encoding: 'utf8', timeout: 15000 }) +validateContext(JSON.parse(docker('context', 'inspect', context))) +const actual = JSON.parse(docker('inspect', container))[0] +validateContainer(actual, { name: container, owner, id: expectedId }) +const port = Number(docker('port', expectedId, '3306/tcp').trim().split(':').at(-1)) +const connection = { + host: '127.0.0.1', + port, + user: 'root', + password: secret, + database: 'ts569_snapshot', + timezone: 'Z' +} + +const open = bigNumberStrings => + knex({ + client: 'mysql2', + connection: { + ...connection, + ...(typeof bigNumberStrings === 'boolean' ? { supportBigNumbers: true, bigNumberStrings } : {}) + }, + pool: { min: 1, max: 1 }, + acquireConnectionTimeout: 5000 + }) +module.exports = { open, ...require('./snapshotJournalNativeFixture.cjs') } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysqlServerCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysqlServerCrash.cjs new file mode 100644 index 000000000..2fd04e10d --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalMysqlServerCrash.cjs @@ -0,0 +1,326 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const receiptPolicy = { receiptLimit: 128, receiptLifetimeMs: 2592000000 } +const assert = require('node:assert/strict'), + { fork } = require('node:child_process'), + { mkdtemp, rm } = require('node:fs/promises'), + { tmpdir } = require('node:os'), + { join } = require('node:path'), + { writeFileSync, readFileSync } = require('node:fs') +const { + open, + seedArchiveClosure, + StorageKnex, + StorageProvider, + exact, + tables +} = require('./snapshotJournalMysqlConnection.cjs') +const { + installSnapshotJournalMysqlGeneration: install, + completeSnapshotJournalMysqlGeneration: complete +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.js') +const { + copySnapshotJournalBootstrapPage: copy +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const ceiling = '9223372036854775807', + intent = 'snapshot_journal_generation' +const { execFileSync } = require('node:child_process') +const { executable, context, validateContainer } = require('./snapshotArchiveDocker.cjs') +const containerId = process.env.TS_STACK_SNAPSHOT_CONTAINER_ID +const docker = (...args) => + execFileSync(executable, ['--context', context, ...args], { + encoding: 'utf8', + timeout: 5000, + env: { ...process.env, MYSQL_PWD: process.env.TS_STACK_SNAPSHOT_MYSQL_SECRET }, + stdio: ['ignore', 'pipe', 'pipe'] + }).trim() +function ownedServer() { + const actual = JSON.parse(docker('inspect', containerId))[0] + validateContainer(actual, { + id: containerId, + name: process.env.TS_STACK_SNAPSHOT_CONTAINER, + owner: process.env.TS_STACK_SNAPSHOT_CONTAINER_OWNER + }) + assert.deepEqual(actual.Config.Entrypoint, ['/bin/sh']) + assert.equal(actual.HostConfig.Memory, 1073741824) + assert.equal(actual.HostConfig.NanoCpus, 2000000000) + assert.equal(actual.HostConfig.Binds, null) + assert.equal(actual.HostConfig.Tmpfs['/var/lib/mysql'], 'rw,nosuid,nodev,size=512m') + const pid = docker('exec', containerId, 'cat', '/var/lib/mysql/fixture.pid') + assert(/^\d+$/.test(pid) && Number(pid) > 1) + assert.equal(docker('exec', containerId, 'cat', '/proc/' + pid + '/comm'), 'mysqld') + return pid +} +function crashServer() { + const pid = ownedServer() + docker('exec', containerId, '/bin/sh', '-c', 'kill -KILL "$1"', 'journal-fixture', pid) + return pid +} +async function ready(previous) { + const deadline = Date.now() + 20000 + let last, recovered + function* attempts() { + while (Date.now() < deadline && recovered === undefined) yield undefined + } + await runInSeries(attempts(), async () => { + try { + docker( + 'exec', + '--env', + 'MYSQL_PWD', + containerId, + 'mysqladmin', + '--protocol=tcp', + '--host=127.0.0.1', + '--user=root', + 'ping' + ) + const pid = ownedServer() + assert.notEqual(pid, previous) + recovered = pid + } catch (error) { + last = error + await new Promise(resolve => setTimeout(resolve, 300)) + } + }) + if (recovered === undefined) throw last + return recovered +} + +async function isolate(k, isolation) { + assert(['READ COMMITTED', 'REPEATABLE READ'].includes(isolation)) + await k.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + const [[level]] = await k.raw('SELECT @@transaction_isolation isolation') + assert.equal(level.isolation.replaceAll('-', ' '), isolation) + await k.raw('SET SESSION innodb_lock_wait_timeout=5') +} +async function bootstrap(k) { + let complete = false + function* pages() { + for (let i = 0; i < 100 && !complete; i++) yield i + } + await runInSeries(pages(), async () => { + const page = await copy(k, 1000000) + assert(!page.invalidated) + complete = page.complete + }) + assert(complete, 'Bootstrap did not complete within its bounded fixture pages') +} +async function finish(k) { + await bootstrap(k) + return await complete(k, ceiling, receiptPolicy) +} + +async function clear(k) { + const [triggers] = await k.raw( + "SELECT TRIGGER_NAME name FROM information_schema.TRIGGERS WHERE TRIGGER_SCHEMA=DATABASE() AND LEFT(TRIGGER_NAME,17)='snapshot_journal_'" + ) + await runInSeries(triggers, async row => { + await k.raw('DROP TRIGGER ??', [row.name]) + }) + const [names] = await k.raw( + "SELECT TABLE_NAME name FROM information_schema.TABLES WHERE TABLE_SCHEMA=DATABASE() AND LEFT(TABLE_NAME,17)='snapshot_journal_'" + ) + await runInSeries(names, async row => { + await k.schema.dropTable(row.name) + }) +} +async function rows(k) { + const result = {} + await runInSeries(tables, async name => { + result[name] = (await k(name).select('*')).map(row => JSON.stringify(row)).sort() + }) + return result +} +async function child() { + process.once('disconnect', () => process.exit(1)) + const childDeadline = setTimeout(() => process.exit(1), 20000) + childDeadline.unref() + const k = open(), + boundary = process.argv[3], + marker = process.argv[4], + isolation = process.argv[5] + let completing = false + const park = phase => { + if (boundary === phase) { + const serverPid = crashServer() + writeFileSync(marker, JSON.stringify({ phase, serverPid })) + process.kill(process.pid, 'SIGKILL') + } + } + const object = sql => /^CREATE (?:TABLE|TRIGGER) (snapshot_journal_\w+)/.exec(sql)?.[1] + k.on('query', q => { + const name = object(q.sql) + if (name) park('before-' + name) + if (completing && q.sql === 'COMMIT;') park('complete-before-commit') + }) + k.on('query-response', (_r, q) => { + const name = object(q.sql) + if (name) park('after-' + name) + if (q.sql.startsWith('update `snapshot_journal_generation` set `nextObject`')) park('after-ack-' + q.bindings[0]) + if (completing && q.sql === 'COMMIT;') park('complete-after-commit') + if (q.sql.startsWith('update `snapshot_journal_bootstrap` set `rowLimit`')) park('bootstrap-after-budget-bind') + if (q.sql.startsWith('insert into `snapshot_journal_physical`')) park('bootstrap-after-metadata') + if (q.sql.startsWith('update `snapshot_journal_bootstrap` set `cursor`')) park('bootstrap-after-progress') + }) + try { + await isolate(k, isolation) + await install(k, ceiling, receiptPolicy) + if (boundary.startsWith('bootstrap-')) { + await copy(k, 1000000) + park('bootstrap-after-commit') + } + if (boundary.startsWith('complete-')) { + await bootstrap(k) + completing = true + await complete(k, ceiling, receiptPolicy) + park('complete-after-commit') + } + throw new Error('Crash boundary missed: ' + boundary) + } finally { + await k.destroy() + } +} +async function killAt(boundary, marker, isolation) { + const p = fork(__filename, ['child', boundary, marker, isolation], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'] + }) + let stderr = '' + p.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-6000) + }) + const timer = setTimeout(() => p.kill('SIGKILL'), 20000), + result = await new Promise((resolve, reject) => { + p.once('error', reject) + p.once('exit', (code, signal) => resolve({ code, signal })) + }) + clearTimeout(timer) + assert.equal(result.signal, 'SIGKILL', stderr) + const evidence = JSON.parse(readFileSync(marker, 'utf8')) + assert.equal(evidence.phase, boundary) + return { previousServerPid: evidence.serverPid, newServerPid: await ready(evidence.serverPid) } +} +async function main() { + let k = open(), + source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + const directory = await mkdtemp(join(tmpdir(), 'ts569-mysql-server-crash-')), + results = [] + try { + ownedServer() + await source.migrate('journal server crash fixture', 'synthetic-journal-server-crash') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)), + { user: foreign } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, foreign.userId) + const baseline = await rows(k) + await source.destroy() + source = undefined + k = undefined + const boundaries = [ + 'after-snapshot_journal_generation', + 'after-snapshot_journal_clock', + 'after-snapshot_journal_retention', + 'after-snapshot_journal_receipts', + 'after-snapshot_journal_scope_0_INSERT', + 'after-snapshot_journal_physical_12_DELETE', + 'bootstrap-after-budget-bind', + 'bootstrap-after-metadata', + 'bootstrap-after-progress', + 'bootstrap-after-commit', + 'complete-before-commit', + 'complete-after-commit' + ] + await runInSeries(['READ COMMITTED', 'REPEATABLE READ'], async isolation => { + await runInSeries(boundaries, async boundary => { + const restarted = await killAt( + boundary, + join(directory, isolation.replaceAll(' ', '-') + '-' + boundary), + isolation + ) + k = open() + await isolate(k, isolation) + const saved = await k(intent).first() + assert(saved) + if (boundary.startsWith('complete-')) assert.equal(saved.complete, boundary === 'complete-after-commit' ? 1 : 0) + if (boundary.startsWith('bootstrap-')) { + const committed = boundary === 'bootstrap-after-commit', + progress = await k('snapshot_journal_bootstrap').first() + assert.equal(progress.rowsUsed, committed ? baseline.transactions.length : 0) + assert.equal(progress.rowLimit, committed ? 1000000 : null) + assert.equal( + progress.cursor, + committed + ? JSON.stringify([Math.max(...baseline.transactions.map(text => JSON.parse(text).transactionId))]) + : null + ) + } + const resumed = await install(k, ceiling, receiptPolicy) + assert.equal(resumed.epoch, saved.epoch) + await finish(k) + let charged = 0 + await runInSeries( + [ + ...tables, + 'snapshot_profile_keys', + 'snapshot_relation_keys', + 'snapshot_certificate_field_keys', + 'snapshot_global_keys' + ], + async table => { + charged += Number((await k(table).count('* AS n').first()).n) + } + ) + assert.equal((await k('snapshot_journal_bootstrap').first()).rowsUsed, charged) + await exact(k) + assert.deepEqual(await rows(k), baseline) + await k('tx_labels').where('txLabelId', 1).update({ label: 'post-server-recovery writer' }) + await exact(k) + await k('tx_labels') + .where('txLabelId', 1) + .update({ + label: JSON.parse(baseline.tx_labels.find(row => JSON.parse(row).txLabelId === 1)).label + }) + await exact(k) + assert.deepEqual(await rows(k), baseline) + await clear(k) + await k.destroy() + k = undefined + results.push({ + isolation, + boundary, + ...restarted, + sourcePreserved: true, + epochPreserved: true, + bootstrapExact: true, + writerRecovered: true + }) + console.log(JSON.stringify(results.at(-1))) + }) + }) + console.log( + JSON.stringify({ + status: 'isolated MySQL server-process crash recovery', + serverCrashes: results.length, + results, + readerAdvertised: false, + limitations: [ + 'tmpfs-backed single MySQL8.4 server; no machine power-loss,replication,PXC/failover or production performance acceptance', + 'this server-crash fixture does not qualify capture; registered migration, floor/quota and receiver remain incomplete' + ] + }) + ) + } finally { + if (source) await source.destroy() + else if (k) await k.destroy() + await rm(directory, { recursive: true, force: true }) + } +} +if (process.argv[2] === 'child') + child().catch(e => { + console.error(e) + process.exitCode = 1 + }) +else + main().catch(e => { + console.error(e) + process.exitCode = 1 + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalNativeFixture.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalNativeFixture.cjs new file mode 100644 index 000000000..1cb496810 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalNativeFixture.cjs @@ -0,0 +1,229 @@ +// Synthetic native fixtures and an independent source-table ownership oracle. +// This fixture does not use production journal generators to compute expected rows. +const assert = require('node:assert/strict') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const identity = '02' + '11'.repeat(32) + +async function seedArchiveClosure(source, userId, otherId) { + const instant = '2026-01-01T00:00:00.000Z' + const date = source.knex.client.config.client === 'mysql2' ? new Date(instant) : instant + const timestamp = { created_at: date, updated_at: date } + const k = source.knex + await k('output_baskets').del() + await runInSeries([1, 2, 3], async id => { + const profileId = id === 2 ? otherId : userId + await k('proven_txs').insert({ + ...timestamp, + provenTxId: id, + txid: String(id).repeat(64), + height: id, + index: 0, + merklePath: Buffer.from([id, 0, 255]), + rawTx: Buffer.from([id, 1, 255]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + await k('transactions').insert({ + ...timestamp, + transactionId: id, + userId: profileId, + provenTxId: id === 3 ? null : id, + status: 'completed', + reference: `tx-${id}`, + isOutgoing: true, + satoshis: 0, + description: `tx-${id}`, + txid: String(id).repeat(64), + rawTx: Buffer.from([id, 2, 255]), + inputBEEF: Buffer.from([id, 3, 255]) + }) + await k('proven_tx_reqs').insert({ + ...timestamp, + provenTxReqId: id, + provenTxId: id, + txid: String(id).repeat(64), + status: 'completed', + attempts: 0, + notified: true, + history: '{}', + notify: '{}', + rawTx: Buffer.from([id, 4, 255]), + wasBroadcast: true + }) + await k('output_baskets').insert({ + ...timestamp, + basketId: id, + userId: profileId, + name: `basket-${id}`, + isDeleted: id === 3 + }) + await k('outputs').insert({ + ...timestamp, + outputId: id, + userId: profileId, + transactionId: id, + basketId: id, + spendable: false, + change: true, + vout: 0, + satoshis: 1, + providedBy: 'you', + purpose: '', + type: 'P2PKH', + lockingScript: Buffer.from([id, 5, 255]) + }) + await k('commissions').insert({ + ...timestamp, + commissionId: id, + userId: profileId, + transactionId: id, + satoshis: 0, + keyOffset: 'offset', + isRedeemed: true, + lockingScript: Buffer.from([id, 6, 255]) + }) + await k('output_tags').insert({ + ...timestamp, + outputTagId: id, + userId: profileId, + tag: `tag-${id}`, + isDeleted: id === 3 + }) + await k('output_tags_map').insert({ + ...timestamp, + outputTagId: id, + outputId: id, + isDeleted: id === 3 + }) + await k('tx_labels').insert({ + ...timestamp, + txLabelId: id, + userId: profileId, + label: `label-${id}`, + isDeleted: id === 3 + }) + await k('tx_labels_map').insert({ + ...timestamp, + txLabelId: id, + transactionId: id, + isDeleted: id === 3 + }) + await k('certificates').insert({ + ...timestamp, + certificateId: id, + userId: profileId, + serialNumber: `serial-${id}`, + type: 'type', + certifier: identity, + subject: identity, + revocationOutpoint: 'a'.repeat(64) + '.0', + signature: 'signature', + isDeleted: id === 3 + }) + await runInSeries(['a', 'Z', 'é', '😀'], async fieldName => { + await k('certificate_fields').insert({ + ...timestamp, + certificateId: id, + userId: profileId, + fieldName, + fieldValue: `value-${id}`, + masterKey: 'key' + }) + }) + await k('sync_states').insert({ + ...timestamp, + syncStateId: id, + userId: profileId, + storageIdentityKey: `peer-${id}`, + storageName: `peer-${id}`, + status: 'unknown', + init: true, + refNum: `state-${id}`, + syncMap: '{}', + when: date + }) + }) + // Composite positions must handle repeated first keys and preserve deleted mappings. + await k('output_tags_map').insert({ ...timestamp, outputTagId: 1, outputId: 3, isDeleted: true }) + await k('tx_labels_map').insert({ ...timestamp, txLabelId: 1, transactionId: 3, isDeleted: true }) +} + +const profiles = [ + ['transactions', 'transactionId'], + ['outputs', 'outputId'], + ['certificates', 'certificateId'], + ['tx_labels', 'txLabelId'], + ['output_baskets', 'basketId'], + ['output_tags', 'outputTagId'], + ['commissions', 'commissionId'], + ['sync_states', 'syncStateId'] +] +const numeric = [...profiles, ['proven_txs', 'provenTxId'], ['proven_tx_reqs', 'provenTxReqId']] +const tables = [...numeric.map(([table]) => table), 'tx_labels_map', 'output_tags_map', 'certificate_fields'] +const quote = value => '`' + value.replaceAll('`', '``') + '`' +const binaryType = k => (k.client.config.client === 'mysql2' ? 'BINARY' : 'BLOB') +const empty = k => "CAST('' AS " + binaryType(k) + ')' +const rawRows = async (k, sql) => (k.client.config.client === 'mysql2' ? (await k.raw(sql))[0] : await k.raw(sql)) +const tuple = (k, table, p) => { + const key = numeric.find(([name]) => name === table)?.[1] + if (key) return [p + '.' + quote(key), '0', empty(k)] + if (table === 'tx_labels_map') return [p + '.txLabelId', p + '.transactionId', empty(k)] + if (table === 'output_tags_map') return [p + '.outputTagId', p + '.outputId', empty(k)] + return [p + '.certificateId', '0', 'CAST(' + p + '.fieldName AS ' + binaryType(k) + ')'] +} +const normalized = (rows, scope) => + rows + .map(row => + JSON.stringify([ + Number(row.tableId), + ...(scope ? [Number(row.userId)] : []), + Number(row.id1), + Number(row.id2), + Buffer.from(row.exactText).toString('hex') + ]) + ) + .sort() +async function expected(k) { + const selections = profiles.map( + ([table, key], id) => + `SELECT ${id} tableId,userId,${quote(key)} id1,0 id2,${empty(k)} exactText FROM ${quote(table)}` + ) + for (const [id, table, left, leftKey, right, rightKey] of [ + [10, 'tx_labels_map', 'tx_labels', 'txLabelId', 'transactions', 'transactionId'], + [11, 'output_tags_map', 'output_tags', 'outputTagId', 'outputs', 'outputId'] + ]) { + for (const [parent, key] of [ + [left, leftKey], + [right, rightKey] + ]) + selections.push( + `SELECT ${id},p.userId,m.${leftKey},m.${rightKey},${empty(k)} FROM ${table} m JOIN ${parent} p ON m.${key}=p.${key}` + ) + } + selections.push( + `SELECT 12,userId,certificateId,0,CAST(fieldName AS ${binaryType(k)}) FROM certificate_fields`, + `SELECT 12,c.userId,f.certificateId,0,CAST(f.fieldName AS ${binaryType(k)}) FROM certificate_fields f JOIN certificates c ON c.certificateId=f.certificateId`, + `SELECT 9,t.userId,r.provenTxReqId,0,${empty(k)} FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid`, + `SELECT 8,t.userId,p.provenTxId,0,${empty(k)} FROM transactions t JOIN proven_txs p ON p.provenTxId=t.provenTxId`, + `SELECT 8,t.userId,p.provenTxId,0,${empty(k)} FROM transactions t JOIN proven_tx_reqs r ON r.txid=t.txid JOIN proven_txs p ON p.provenTxId=r.provenTxId` + ) + return await rawRows(k, selections.join(' UNION ')) +} +async function exact(k) { + assert.deepEqual( + normalized(await k('snapshot_journal_scope').where('present', 1), true), + normalized(await expected(k), true) + ) + const physical = await k('snapshot_journal_physical').where('present', 1) + const selections = tables.map((table, tableId) => { + const keys = tuple(k, table, 's') + return `SELECT ${tableId} tableId,${keys[0]} id1,${keys[1]} id2,${keys[2]} exactText FROM ${quote(table)} s` + }) + assert.deepEqual(normalized(physical, false), normalized(await rawRows(k, selections.join(' UNION ALL ')), false)) + for (const row of physical) assert(BigInt(String(row.generation)) > 0n) + if (k.client.config.client === 'mysql2') assert.equal((await k('snapshot_journal_events')).length, 0) +} +module.exports = { knex, StorageKnex, StorageProvider, seedArchiveClosure, tables, exact } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalReceiptMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalReceiptMysql.cjs new file mode 100644 index 000000000..4243587ea --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalReceiptMysql.cjs @@ -0,0 +1,259 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const assert = require('node:assert/strict') +const { open } = require('./snapshotJournalMysqlConnection.cjs') +const { + snapshotJournalReceiptDdl, + snapshotJournalReceiptBinding, + recordSnapshotJournalReceipt, + readSnapshotJournalReceipt, + collectSnapshotJournalReceipts +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') +const binding = { + backend: '11'.repeat(32), + epoch: '12345678-1234-4234-9234-123456789012', + source: '22'.repeat(32), + schema: '33'.repeat(32), + storageIdentity: 'synthetic-storage', + identityKey: '02' + '44'.repeat(32), + userId: 1, + chain: 'test' +} +async function qualify(isolation, bigNumberStrings) { + const k = open(bigNumberStrings), + peer = open(bigNumberStrings) + try { + await runInSeries([k, peer], async db => { + await db.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + }) + await runInSeries( + ['snapshot_journal_receipts', 'snapshot_journal_retention', 'receipt_fixture_source'], + async table => { + await k.schema.dropTableIfExists(table) + } + ) + await runInSeries(snapshotJournalReceiptDdl(k), async sql => { + await k.raw(sql) + }) + await k('snapshot_journal_retention').insert({ id: 1, floor: '0', receiptLimit: 2, receiptLifetimeMs: 2592000000 }) + await k.raw('CREATE TABLE receipt_fixture_source(id INTEGER PRIMARY KEY,value INTEGER NOT NULL) ENGINE=InnoDB') + await k('receipt_fixture_source').insert({ id: 1, value: 0 }) + const input = n => ({ + requestId: n.toString(16).padStart(64, '0'), + highWater: '9007199254740993', + expiresAt: Date.now() + 60000 + }) + const a = input(1), + b = input(2), + c = input(3) + const stored = await k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + assert.deepEqual(stored, { ...a, binding: snapshotJournalReceiptBinding(binding), floor: '0' }) + assert.deepEqual(await peer.transaction(t => readSnapshotJournalReceipt(t, binding, a)), stored) + assert.deepEqual(await k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)), stored) + await assert.rejects(k.transaction(t => readSnapshotJournalReceipt(t, { ...binding, userId: 2 }, a))) + await assert.rejects( + k.transaction(async t => { + await recordSnapshotJournalReceipt(t, binding, b) + throw new Error('before-commit') + }), + /before-commit/ + ) + assert.equal((await k('snapshot_journal_receipts')).length, 1) + await assert.rejects( + (async () => { + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, b)) + throw new Error('lost-ack') + })(), + /lost-ack/ + ) + assert.equal((await peer.transaction(t => readSnapshotJournalReceipt(t, binding, b))).requestId, b.requestId) + await assert.rejects(k.transaction(t => recordSnapshotJournalReceipt(t, binding, c))) + const stale = await peer.transaction() + try { + await stale('snapshot_journal_receipts').select('requestId') + await k('snapshot_journal_receipts').where('requestId', a.requestId).delete() + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, c)) + await assert.rejects(recordSnapshotJournalReceipt(stale, binding, a)) + } finally { + await stale.rollback() + } + await k('snapshot_journal_receipts').where('requestId', c.requestId).delete() + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, a)) + await k('snapshot_journal_receipts').where('requestId', a.requestId).update({ expiresAt: 1 }) + await assert.rejects(k.transaction(t => recordSnapshotJournalReceipt(t, binding, c))) + assert.equal(await k.transaction(t => collectSnapshotJournalReceipts(t)), 1) + await k.transaction(t => recordSnapshotJournalReceipt(t, binding, c)) + const held = await peer.transaction() + try { + await held('snapshot_journal_retention').where('id', 1).forUpdate().first() + const start = performance.now() + await assert.rejects( + k.transaction(t => recordSnapshotJournalReceipt(t, binding, b)), + e => e.code === 'ER_LOCK_NOWAIT' + ) + await assert.rejects( + k.transaction(t => readSnapshotJournalReceipt(t, binding, b)), + e => e.code === 'ER_LOCK_NOWAIT' + ) + assert(performance.now() - start < 1000) + await k('receipt_fixture_source').where('id', 1).update({ value: 1 }) + } finally { + await held.rollback() + } + await k('snapshot_journal_retention').update({ floor: '9007199254740994' }) + await assert.rejects(peer.transaction(t => readSnapshotJournalReceipt(t, binding, b))) + await k('snapshot_journal_retention').update({ floor: '0' }) + await k('snapshot_journal_receipts').delete() + await assert.rejects(peer.transaction(t => readSnapshotJournalReceipt(t, binding, b))) + const rows = Array.from({ length: 128 }, (_, n) => ({ + requestId: n.toString(16).padStart(64, '0'), + binding: snapshotJournalReceiptBinding(binding), + highWater: '9', + floor: '0', + expiresAt: 1 + })) + await k('snapshot_journal_retention').update({ receiptLimit: 128 }) + await k('snapshot_journal_receipts').insert(rows) + assert.equal(await k.transaction(t => collectSnapshotJournalReceipts(t)), 64) + assert.equal(await k.transaction(t => collectSnapshotJournalReceipts(t)), 64) + assert.equal(await k.transaction(t => collectSnapshotJournalReceipts(t)), 0) + return { + isolation, + bigNumberStrings, + exactSigned63: true, + independentRead: true, + immutableRetry: true, + rollbackAndLostAck: true, + boundedCapacityAndCollector: true, + nonwaitingRetentionLock: true, + currentReadAfterOlderSnapshot: true, + independentSourceWrite: true, + collectedOrMissingPrefixRefused: true + } + } finally { + await Promise.all([k.destroy(), peer.destroy()]) + } +} +const { fork } = require('node:child_process'), + { mkdtemp, rm } = require('node:fs/promises'), + { writeFileSync, readFileSync } = require('node:fs'), + { tmpdir } = require('node:os'), + { join } = require('node:path') +const processRequest = { requestId: 'ee'.repeat(32), highWater: '9007199254740993', expiresAt: 0 } +async function crashChild() { + process.once('disconnect', () => process.exit(1)) + const deadline = setTimeout(() => process.exit(1), 15000) + deadline.unref() + const [phase, marker, isolation, expiresAt] = process.argv.slice(3), + k = open() + const die = () => { + writeFileSync(marker, phase) + process.kill(process.pid, 'SIGKILL') + } + try { + assert(['READ COMMITTED', 'REPEATABLE READ'].includes(isolation)) + await k.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + await k.transaction(async t => { + if (phase.startsWith('record')) + await recordSnapshotJournalReceipt(t, binding, { ...processRequest, expiresAt: Number(expiresAt) }) + else assert.equal(await collectSnapshotJournalReceipts(t), 1) + if (phase.endsWith('before-commit')) die() + }) + if (phase.endsWith('after-commit')) die() + throw new Error('Unreached owned crash boundary') + } finally { + await k.destroy() + } +} +async function processLoss() { + const k = open(), + directory = await mkdtemp(join(tmpdir(), 'ts569-receipt-loss-')), + results = [] + try { + await runInSeries(['READ COMMITTED', 'REPEATABLE READ'], async isolation => { + await runInSeries( + ['record-before-commit', 'record-after-commit', 'collect-before-commit', 'collect-after-commit'], + async phase => { + await k('snapshot_journal_receipts').delete() + await k('snapshot_journal_retention').update({ floor: '0', receiptLimit: 128, receiptLifetimeMs: 2592000000 }) + const request = { ...processRequest, expiresAt: Date.now() + 60000 }, + marker = join(directory, results.length + '.txt') + if (phase.startsWith('collect')) + await k('snapshot_journal_receipts').insert({ + ...request, + binding: snapshotJournalReceiptBinding(binding), + floor: '0', + expiresAt: 1 + }) + const child = fork(__filename, ['child', phase, marker, isolation, String(request.expiresAt)], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'] + }) + let stderr = '' + child.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-6000) + }) + const timer = setTimeout(() => child.kill('SIGKILL'), 15000) + let terminal + try { + terminal = await new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + } finally { + clearTimeout(timer) + } + assert.equal(terminal.signal, 'SIGKILL', stderr) + assert.equal(readFileSync(marker, 'utf8'), phase) + const exists = phase === 'record-after-commit' || phase === 'collect-before-commit' + assert.equal((await k('snapshot_journal_receipts')).length, exists ? 1 : 0) + if (phase === 'record-after-commit') + assert.equal( + (await k.transaction(t => readSnapshotJournalReceipt(t, binding, request))).highWater, + request.highWater + ) + if (phase === 'record-before-commit') + await assert.rejects(k.transaction(t => readSnapshotJournalReceipt(t, binding, request))) + await k('receipt_fixture_source') + .where('id', 1) + .update({ value: results.length + 2 }) + results.push({ + isolation, + phase, + signal: terminal.signal, + receiptPersisted: exists, + sourceWriteAfterLoss: true + }) + } + ) + }) + return results + } finally { + await k.destroy() + await rm(directory, { recursive: true, force: true }) + } +} +async function main() { + const results = [] + await runInSeries(['READ COMMITTED', 'REPEATABLE READ'], async isolation => { + await runInSeries([false, true], async bigNumberStrings => { + results.push(await qualify(isolation, bigNumberStrings)) + }) + }) + const crashes = await processLoss() + console.log( + JSON.stringify({ + status: 'native receipt-store and process-loss draft', + results, + crashes, + productionAdoption: false, + limitations: [ + 'Synthetic receipt transactions do not establish full source-capture ordering', + 'This synthetic receipt fixture does not qualify the separate capture controller; floor/tombstone lifecycle and registered migration remain incomplete', + 'A receipt cannot reopen a killed retained read view' + ] + }) + ) +} +;(process.argv[2] === 'child' ? crashChild() : main()).catch(e => { + console.error(e) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionChild.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionChild.cjs new file mode 100644 index 000000000..eddde7fa5 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionChild.cjs @@ -0,0 +1,47 @@ +const assert = require('node:assert/strict') +const input = JSON.parse(process.argv[2]) +const { advanceSnapshotJournalFloor } = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') +const { + collectSnapshotJournalTombstones +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalCollection.js') +const { inject } = require('./snapshotJournalRetentionProcessLoss.cjs') +async function main() { + assert(['sqlite', 'mysql'].includes(input.backend)) + const k = + input.backend === 'mysql' + ? require('./snapshotJournalMysqlConnection.cjs').open(true) + : require('knex').knex({ + client: 'better-sqlite3', + connection: { filename: input.filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + process.once('disconnect', () => process.exit(1)) + const deadline = setTimeout(() => process.exit(2), 20000) + deadline.unref() + try { + if (input.backend === 'sqlite') await k.raw('PRAGMA busy_timeout=0') + else { + assert(['READ COMMITTED', 'REPEATABLE READ'].includes(input.isolation)) + await k.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + input.isolation) + } + inject(k, input.operation, input.phase) + await k.transaction(t => + input.operation === 'floor' + ? advanceSnapshotJournalFloor(t, input.floor) + : collectSnapshotJournalTombstones(t, input.request) + ) + throw new Error('Did not reach retention process-loss boundary') + } catch (error) { + try { + await k.destroy() + } catch (cleanupError) { + throw new AggregateError([error, cleanupError], 'Retention child cleanup failed') + } + throw error + } +} +main().catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionCuts.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionCuts.cjs new file mode 100644 index 000000000..42749fdcc --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionCuts.cjs @@ -0,0 +1,133 @@ +const assert = require('node:assert/strict') +const { join } = require('node:path') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { tables, exact } = require('./snapshotJournalNativeFixture.cjs') +const { killAt } = require('./snapshotJournalRetentionProcessLoss.cjs') +const { + advanceSnapshotJournalFloor, + collectSnapshotJournalReceipts +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') +const { + collectSnapshotJournalTombstones +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalCollection.js') +const { snapshotArchiveDatabaseNow } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveSql.js') +const { snapshotJournalRevisionText } = require('../../out/src/storage/snapshot/journal/SnapshotJournalRevisionSql.js') +async function source(k) { + const rows = {} + await runInSeries(tables, async table => { + rows[table] = await k(table) + }) + return rows +} +async function newest(k) { + let prefix = '0' + await runInSeries(['scope', 'physical'], async stream => { + const table = 'snapshot_journal_' + stream + const row = await k(table) + .select({ revisionText: snapshotJournalRevisionText(k, 'revision') }) + .orderBy(table + '.revision', 'desc') + .first() + if (row !== undefined && BigInt(row.revisionText) > BigInt(prefix)) prefix = row.revisionText + }) + assert(BigInt(prefix) > 0n) + return prefix +} +module.exports = async function cuts(k, input) { + await k('snapshot_journal_receipts').update({ expiresAt: (await snapshotArchiveDatabaseNow(k)) - 1 }) + await k.transaction(t => collectSnapshotJournalReceipts(t)) + assert.equal((await k('snapshot_journal_receipts')).length, 0) + const results = [] + const clock = async () => + ( + await k('snapshot_journal_clock') + .select({ revision: snapshotJournalRevisionText(k, 'revision') }) + .first() + ).revision + await runInSeries(['before-commit', 'after-commit'], async phase => { + await k('tx_labels') + .where('txLabelId', 1) + .update({ label: 'floor process loss ' + phase }) + const floor = await newest(k), + previous = String((await k('snapshot_journal_retention').first()).floor), + beforeClock = input.backend === 'sqlite' ? await clock() : undefined + const beforeSource = await source(k) + await killAt({ ...input, operation: 'floor', phase, floor, marker: join(input.directory, 'floor-' + phase) }) + assert.equal( + String((await k('snapshot_journal_retention').first()).floor), + phase === 'after-commit' ? floor : previous + ) + if (input.backend === 'sqlite') + assert.equal(await clock(), phase === 'after-commit' ? String(BigInt(beforeClock) + 1n) : beforeClock) + assert.deepEqual(await source(k), beforeSource) + const recovered = await k.transaction(t => advanceSnapshotJournalFloor(t, floor)) + assert.equal(recovered.floor, floor) + await exact(k) + results.push({ operation: 'floor', phase, atomic: true, lostAcknowledgementRecovered: phase === 'after-commit' }) + }) + let firstId = 1000 + await runInSeries(['scope', 'physical'], async stream => { + await runInSeries(['after-first-delete', 'before-commit', 'after-commit'], async phase => { + const ids = [firstId++, firstId++, firstId++], + date = input.backend === 'mysql' ? new Date('2026-01-01T00:00:00Z') : '2026-01-01T00:00:00.000Z' + await k('tx_labels').insert( + ids.map(id => ({ + txLabelId: id, + userId: input.userId, + label: 'gc process loss ' + id, + isDeleted: false, + created_at: date, + updated_at: date + })) + ) + await k('tx_labels').whereIn('txLabelId', ids).delete() + const floor = await newest(k) + await k.transaction(t => advanceSnapshotJournalFloor(t, floor)) + const key = { + tableId: 3, + ...(stream === 'scope' ? { userId: input.userId } : {}), + id1: ids[0] - 1, + id2: 0, + exactText: '' + } + const request = { epoch: input.epoch, floor, stream, limit: 3, after: { epoch: input.epoch, floor, stream, key } } + const table = 'snapshot_journal_' + stream, + order = + stream === 'scope' ? ['tableId', 'userId', 'id1', 'id2', 'exactText'] : ['tableId', 'id1', 'id2', 'exactText'] + const before = await k(table).orderBy(order), + beforeSource = await source(k), + beforeClock = input.backend === 'sqlite' ? await clock() : undefined + const deleted = row => + Number(row.tableId) === 3 && + ids.includes(Number(row.id1)) && + (stream !== 'scope' || Number(row.userId) === input.userId) + assert.equal(before.filter(deleted).length, 3) + assert(before.filter(deleted).every(row => Number(row.present) === 0)) + await killAt({ ...input, operation: stream, phase, request, marker: join(input.directory, stream + '-' + phase) }) + assert.deepEqual( + await k(table).orderBy(order), + phase === 'after-commit' ? before.filter(row => !deleted(row)) : before + ) + if (input.backend === 'sqlite') + assert.equal(await clock(), phase === 'after-commit' ? String(BigInt(beforeClock) + 1n) : beforeClock) + assert.deepEqual(await source(k), beforeSource) + const recovered = await k.transaction(t => collectSnapshotJournalTombstones(t, request)) + assert.equal(recovered.removed, phase === 'after-commit' ? 0 : 3) + assert(recovered.examined <= 3) + await exact(k) + assert.deepEqual(await source(k), beforeSource) + await k('tx_labels') + .where('txLabelId', 1) + .update({ label: 'writer after ' + stream + ' ' + phase }) + await exact(k) + results.push({ + operation: stream, + phase, + atomic: true, + sourcePreserved: true, + recovered: true, + writerProgress: true + }) + }) + }) + return results +} diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysql.cjs new file mode 100644 index 000000000..95890c5b8 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysql.cjs @@ -0,0 +1,201 @@ +const assert = require('node:assert/strict') +const { mkdtemp, rm } = require('node:fs/promises') +const { tmpdir } = require('node:os') +const { join } = require('node:path') +const cuts = require('./snapshotJournalRetentionCuts.cjs') +const { + open, + StorageKnex, + StorageProvider, + seedArchiveClosure, + tables, + exact +} = require('./snapshotJournalMysqlConnection.cjs') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + installSnapshotJournalMysqlGeneration, + completeSnapshotJournalMysqlGeneration +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.js') +const { + copySnapshotJournalBootstrapPage +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const { + advanceSnapshotJournalFloor, + collectSnapshotJournalReceipts +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') +const { + collectSnapshotJournalTombstones, + snapshotJournalCollectionQuery +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalCollection.js') +const { snapshotArchiveDatabaseNow } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveSql.js') +const identity = '02' + '11'.repeat(32) +const request = { ceiling: '9223372036854775807', receiptPolicy: { receiptLimit: 128, receiptLifetimeMs: 600000 } } +async function collect(k, epoch, floor, stream) { + let after, + complete = false, + examined = 0, + removed = 0 + function* pages() { + for (let n = 0; n < 100 && !complete; n++) yield n + } + await runInSeries(pages(), async () => { + const input = { epoch, floor, stream, limit: 32, after } + const result = await k.transaction(t => collectSnapshotJournalTombstones(t, input)) + assert(result) + assert(result.examined <= 32) + if (after !== undefined) { + const sql = snapshotJournalCollectionQuery(k, input).toSQL() + const [plan] = await k.raw('EXPLAIN ' + sql.sql, sql.bindings) + assert.equal(plan[0].key, 'PRIMARY') + assert.equal(plan[0].type, 'range') + assert(!String(plan[0].Extra).includes('filesort')) + } + examined += result.examined + removed += result.removed + after = result.after + complete = result.complete + }) + assert(complete, 'bounded collector did not finish in one hundred pages') + return { examined, removed } +} +module.exports = async function main(isolation) { + assert(['READ COMMITTED', 'REPEATABLE READ'].includes(isolation)) + const directory = await mkdtemp(join(tmpdir(), 'ts569-retention-mysql-')) + const k = open(true), + peer = open(true) + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + let first, next, failure + try { + await storage.migrate('native retention fixture', 'synthetic-retention-native') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identity), + { user: other } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(storage, user.userId, other.userId) + await installSnapshotJournalMysqlGeneration(k, request.ceiling, request.receiptPolicy) + let finished = false + function* bootstrap() { + for (let n = 0; n < 100 && !finished; n++) yield n + } + await runInSeries(bootstrap(), async () => { + finished = (await copySnapshotJournalBootstrapPage(k, 1000000)).complete + }) + assert(finished) + await completeSnapshotJournalMysqlGeneration(k, request.ceiling, request.receiptPolicy) + const date = new Date('2026-01-01T00:00:00Z') + const labels = Array.from({ length: 600 }, (_, n) => ({ + txLabelId: n + 100, + userId: user.userId, + label: 'native-retention-' + n, + isDeleted: false, + created_at: date, + updated_at: date + })) + await k('tx_labels').insert(labels) + await k.raw('ALTER TABLE snapshot_journal_events AUTO_INCREMENT=9007199254740993') + await peer.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation) + const old = await peer.transaction() + try { + assert.equal((await old('snapshot_journal_receipts')).length, 0) + first = await storage.openSnapshotJournalSource(identity, request) + assert(BigInt(first.receipt.highWater) > 9007199254740991n) + const started = performance.now() + await assert.rejects(advanceSnapshotJournalFloor(old, String(BigInt(first.receipt.highWater) + 1n))) + assert(performance.now() - started < 2000) + } finally { + await old.rollback() + } + const floor = await k.transaction(t => advanceSnapshotJournalFloor(t, first.receipt.highWater)) + assert.equal(floor.floor, first.receipt.highWater) + const held = await peer.transaction() + try { + await held('snapshot_journal_clock').where('id', 1).forUpdate().first() + const started = performance.now() + await assert.rejects( + k.transaction(t => advanceSnapshotJournalFloor(t, first.receipt.highWater)), + error => error.code === 'ER_LOCK_NOWAIT' + ) + assert(performance.now() - started < 2000) + } finally { + await held.rollback() + } + const pinned = (await first.readPage('certificateFields')).rows + await k('certificate_fields').where({ certificateId: 1, fieldName: 'é' }).delete() + assert.deepEqual((await first.readPage('certificateFields')).rows, pinned) + await runInSeries(['scope', 'physical'], async stream => { + const before = await collect(k, first.receiptBinding.epoch, first.receipt.highWater, stream) + assert.equal(before.removed, 0, 'newer tombstones must survive an older floor') + assert(before.examined >= 600) + }) + await first.close() + first = undefined + const now = await snapshotArchiveDatabaseNow(k) + await k('snapshot_journal_receipts').update({ expiresAt: now - 1 }) + assert.equal(await k.transaction(t => collectSnapshotJournalReceipts(t)), 1) + next = await storage.openSnapshotJournalSource(identity, request) + await k.transaction(t => advanceSnapshotJournalFloor(t, next.receipt.highWater)) + const beforeSource = {} + await runInSeries(tables, async table => { + beforeSource[table] = await k(table) + }) + const results = {} + await runInSeries(['scope', 'physical'], async stream => { + results[stream] = await collect(k, next.receiptBinding.epoch, next.receipt.highWater, stream) + assert(results[stream].removed > 0) + assert(results[stream].examined >= 600) + }) + await exact(k) + await runInSeries(tables, async table => { + assert.deepEqual(await k(table), beforeSource[table]) + }) + const epoch = next.receiptBinding.epoch + await next.close() + next = undefined + const processLoss = await cuts(k, { backend: 'mysql', isolation, directory, epoch, userId: user.userId }) + console.log( + JSON.stringify({ + fixture: 'journal-retention-native', + isolation, + beyondSafeInteger: true, + liveReceiptPinned: true, + currentReadAfterOldSnapshot: true, + lockRefusal: true, + retainedViewImmutable: true, + newerTombstonePreserved: true, + databaseClockExpiry: true, + primaryKeyRangePlans: true, + bound: 32, + exactLiveMetadata: true, + allThirteenSourceTablesPreserved: true, + results, + processLoss, + limitations: [ + 'internal component fixture', + 'complete mutation and source integration pending', + 'no runtime quotas or registered migration', + 'synthetic MySQL8.4; no production/PXC/failover qualification' + ] + }) + ) + } catch (error) { + failure = { error } + } + const closed = await Promise.allSettled([ + first?.close(), + next?.close(), + storage.destroy(), + k.destroy(), + peer.destroy() + ]) + const errors = closed.filter(result => result.status === 'rejected').map(result => result.reason) + try { + await rm(directory, { recursive: true, force: true }) + } catch (error) { + errors.push(error) + } + if (errors.length) + throw new AggregateError( + [...(failure === undefined ? [] : [failure.error]), ...errors], + 'Native retention fixture cleanup failed' + ) + if (failure !== undefined) throw failure.error +} diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysqlRc.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysqlRc.cjs new file mode 100644 index 000000000..a0377ff37 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysqlRc.cjs @@ -0,0 +1,4 @@ +require('./snapshotJournalRetentionMysql.cjs')('READ COMMITTED').catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysqlRr.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysqlRr.cjs new file mode 100644 index 000000000..6cc9e5de4 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionMysqlRr.cjs @@ -0,0 +1,4 @@ +require('./snapshotJournalRetentionMysql.cjs')('REPEATABLE READ').catch(error => { + console.error(error) + process.exitCode = 1 +}) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionProcessLoss.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionProcessLoss.cjs new file mode 100644 index 000000000..977e08f7b --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionProcessLoss.cjs @@ -0,0 +1,67 @@ +const assert = require('node:assert/strict') +const { fork } = require('node:child_process') +const { writeFileSync } = require('node:fs') +const { open } = require('node:fs/promises') +const { join } = require('node:path') + +// These hooks belong only to the disposable fixture's own native pool. +function inject(k, operation, phase) { + assert(['floor', 'scope', 'physical'].includes(operation)) + assert(['after-first-delete', 'before-commit', 'after-commit'].includes(phase)) + assert(operation !== 'floor' || phase !== 'after-first-delete') + const prefix = + operation === 'floor' ? 'update `snapshot_journal_retention`' : 'delete from `snapshot_journal_' + operation + '`' + let changed = false + const park = () => { + writeFileSync(4, phase) + process.kill(process.pid, 'SIGKILL') + } + k.on('query', q => { + if (q.sql.startsWith(prefix)) changed = true + if (changed && q.sql === 'COMMIT;' && phase === 'before-commit') park() + }) + k.on('query-response', (_response, q) => { + if (q.sql.startsWith(prefix) && phase === 'after-first-delete') park() + }) + const transaction = k.client.transaction + k.client.transaction = function (...parameters) { + const trx = transaction.apply(this, parameters) + const commit = trx.commit + trx.commit = async function (...values) { + const result = await commit.apply(this, values) + if (changed && phase === 'after-commit') { + await this.transactor.executionPromise + park() + } + return result + } + return trx + } +} +async function killAt(input) { + const { marker, ...childInput } = input + const output = await open(marker, 'wx+', 0o600) + const child = fork(join(__dirname, 'snapshotJournalRetentionChild.cjs'), [JSON.stringify(childInput)], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc', output.fd] + }) + let stderr = '' + child.stderr.on('data', data => { + stderr += data + }) + const timer = setTimeout(() => child.kill('SIGKILL'), 20000) + try { + const result = await new Promise((resolve, reject) => { + child.once('error', reject) + child.once('exit', (code, signal) => resolve({ code, signal })) + }) + assert.equal(result.signal, 'SIGKILL', stderr) + const bytes = Buffer.alloc(64) + const { bytesRead } = await output.read(bytes, 0, bytes.length, 0) + assert.equal(bytes.subarray(0, bytesRead).toString('utf8'), input.phase) + } finally { + clearTimeout(timer) + await output.close() + if (child.exitCode === null && child.signalCode === null) child.kill('SIGKILL') + } +} +module.exports = { inject, killAt } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionSqlite.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionSqlite.cjs new file mode 100644 index 000000000..90376da6c --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalRetentionSqlite.cjs @@ -0,0 +1,209 @@ +const assert = require('node:assert/strict') +const { mkdtemp, rm } = require('node:fs/promises') +const { tmpdir } = require('node:os') +const { join } = require('node:path') +const { + knex, + StorageKnex, + StorageProvider, + seedArchiveClosure, + tables, + exact +} = require('./snapshotJournalNativeFixture.cjs') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + installSnapshotJournalSqliteGeneration, + completeSnapshotJournalSqliteGeneration +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.js') +const { + copySnapshotJournalBootstrapPage +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const { + advanceSnapshotJournalFloor, + collectSnapshotJournalReceipts +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalReceipt.js') +const { + collectSnapshotJournalTombstones, + snapshotJournalCollectionQuery +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalCollection.js') +const { snapshotArchiveDatabaseNow } = require('../../out/src/storage/snapshot/archive/SnapshotArchiveSql.js') +const cuts = require('./snapshotJournalRetentionCuts.cjs') +const identity = '02' + '11'.repeat(32) +const request = { ceiling: '9223372036854775807', receiptPolicy: { receiptLimit: 128, receiptLifetimeMs: 600000 } } +const open = filename => + knex({ client: 'better-sqlite3', connection: { filename }, useNullAsDefault: true, pool: { min: 1, max: 1 } }) +async function collect(k, epoch, floor, stream) { + let after, + complete = false, + examined = 0, + removed = 0 + function* pages() { + for (let n = 0; n < 100 && !complete; n++) yield n + } + await runInSeries(pages(), async () => { + const input = { epoch, floor, stream, limit: 32, after }, + result = await k.transaction(t => collectSnapshotJournalTombstones(t, input)) + assert(result) + assert(result.examined <= 32) + if (after !== undefined) { + const sql = snapshotJournalCollectionQuery(k, input).toSQL(), + plan = await k.raw('EXPLAIN QUERY PLAN ' + sql.sql, sql.bindings) + assert(plan.some(row => row.detail.includes('SEARCH j USING INDEX sqlite_autoindex_snapshot_journal_'))) + assert(plan.every(row => !row.detail.includes('TEMP B-TREE'))) + } + examined += result.examined + removed += result.removed + after = result.after + complete = result.complete + }) + assert(complete) + return { examined, removed } +} +async function main() { + const directory = await mkdtemp(join(tmpdir(), 'ts569-retention-sqlite-')), + filename = join(directory, 'wallet.sqlite'), + k = open(filename), + peer = open(filename) + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + let first, next, failure + try { + await k.raw('PRAGMA journal_mode=WAL') + await k.raw('PRAGMA busy_timeout=0') + await peer.raw('PRAGMA busy_timeout=0') + await storage.migrate('native retention WAL fixture', 'synthetic-retention-native') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identity), + { user: other } = await storage.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(storage, user.userId, other.userId) + await installSnapshotJournalSqliteGeneration(k, request.ceiling, request.receiptPolicy) + let finished = false + function* bootstrap() { + for (let n = 0; n < 100 && !finished; n++) yield n + } + await runInSeries(bootstrap(), async () => { + finished = (await copySnapshotJournalBootstrapPage(k, 1000000)).complete + }) + assert(finished) + await completeSnapshotJournalSqliteGeneration(k, request.receiptPolicy) + const date = '2026-01-01T00:00:00.000Z' + await runInSeries( + Array.from({ length: 6 }, (_, n) => n), + async n => { + await k('tx_labels').insert( + Array.from({ length: 100 }, (_, i) => ({ + txLabelId: 100 + n * 100 + i, + userId: user.userId, + label: 'native-retention-' + (n * 100 + i), + isDeleted: false, + created_at: date, + updated_at: date + })) + ) + } + ) + await k('snapshot_journal_clock').where('id', 1).update({ revision: '9007199254740993' }) + first = await storage.openSnapshotJournalSource(identity, request) + assert(BigInt(first.receipt.highWater) > 9007199254740991n) + await assert.rejects( + k.transaction(t => advanceSnapshotJournalFloor(t, String(BigInt(first.receipt.highWater) + 1n))) + ) + const floor = await k.transaction(t => advanceSnapshotJournalFloor(t, first.receipt.highWater)) + assert.equal(floor.floor, first.receipt.highWater) + const held = await peer.transaction() + try { + await held('snapshot_journal_clock') + .where('id', 1) + .update({ revision: held.ref('revision') }) + const started = performance.now() + await assert.rejects( + k.transaction(t => advanceSnapshotJournalFloor(t, first.receipt.highWater)), + error => error.code === 'SQLITE_BUSY' + ) + assert(performance.now() - started < 2000) + } finally { + await held.rollback() + } + const pinned = await first.readPage('certificateFields') + await k('certificate_fields').where({ certificateId: 1, fieldName: 'é' }).delete() + assert.deepEqual(await first.readPage('certificateFields'), pinned) + const before = await collect(k, first.receiptBinding.epoch, floor.floor, 'scope') + assert(before.examined >= 600) + assert.equal(before.removed, 0) + const beforePhysical = await collect(k, first.receiptBinding.epoch, floor.floor, 'physical') + assert(beforePhysical.examined >= 600) + assert.equal(beforePhysical.removed, 0) + await first.close() + first = undefined + await k('snapshot_journal_receipts').update({ expiresAt: (await snapshotArchiveDatabaseNow(k)) - 1 }) + assert.equal(await k.transaction(t => collectSnapshotJournalReceipts(t)), 1) + next = await storage.openSnapshotJournalSource(identity, request) + const advanced = await k.transaction(t => advanceSnapshotJournalFloor(t, next.receipt.highWater)), + beforeSource = {} + await runInSeries(tables, async table => { + beforeSource[table] = await k(table) + }) + const results = { + scope: await collect(k, next.receiptBinding.epoch, advanced.floor, 'scope'), + physical: await collect(k, next.receiptBinding.epoch, advanced.floor, 'physical') + } + assert(results.scope.removed > 0) + assert(results.physical.removed > 0) + await exact(k) + await runInSeries(tables, async table => { + assert.deepEqual(await k(table), beforeSource[table]) + }) + const epoch = next.receiptBinding.epoch + await next.close() + next = undefined + const processLoss = await cuts(k, { backend: 'sqlite', filename, directory, epoch, userId: user.userId }) + assert.deepEqual(await k.raw('PRAGMA foreign_key_check'), []) + console.log( + JSON.stringify({ + fixture: 'journal-retention-WAL', + beyondSafeInteger: true, + liveReceiptPinned: true, + lockRefusal: true, + retainedViewImmutable: true, + databaseClockExpiry: true, + primaryKeyRangePlans: true, + bound: 32, + allThirteenSourceTablesPreserved: true, + results, + processLoss, + limitations: [ + 'internal component fixture', + 'complete mutation and source integration pending', + 'no runtime quotas or registered migration', + 'no production or filesystem power-loss qualification' + ] + }) + ) + } catch (error) { + failure = { error } + } + const closed = await Promise.allSettled([ + first?.close(), + next?.close(), + storage.destroy(), + k.destroy(), + peer.destroy() + ]), + errors = closed.filter(result => result.status === 'rejected').map(result => result.reason) + try { + await rm(directory, { recursive: true, force: true }) + } catch (error) { + errors.push(error) + } + if (errors.length) + throw new AggregateError( + [...(failure === undefined ? [] : [failure.error]), ...errors], + 'WAL retention fixture cleanup failed' + ) + if (failure !== undefined) throw failure.error +} +module.exports = main +if (require.main?.filename === __filename) + main().catch(error => { + console.error(error) + process.exitCode = 1 + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotJournalSqliteCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalSqliteCrash.cjs new file mode 100644 index 000000000..13bb02326 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotJournalSqliteCrash.cjs @@ -0,0 +1,221 @@ +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const receiptPolicy = { receiptLimit: 128, receiptLifetimeMs: 2592000000 } +const assert = require('node:assert/strict'), + { fork } = require('node:child_process'), + { mkdtemp, rm } = require('node:fs/promises'), + { readFileSync, writeFileSync } = require('node:fs'), + { tmpdir } = require('node:os'), + { join } = require('node:path') +const { knex, StorageKnex, StorageProvider, tables, exact } = require('./snapshotJournalNativeFixture.cjs'), + { seedArchiveClosure } = require('./snapshotJournalNativeFixture.cjs') +const { + installSnapshotJournalSqliteGeneration: install, + readSnapshotJournalSqliteGeneration: read, + completeSnapshotJournalSqliteGeneration: complete +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.js') +const { + copySnapshotJournalBootstrapPage: copy +} = require('../../out/src/storage/snapshot/journal/SnapshotJournalBootstrap.js') +const open = filename => + knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) +const finish = async k => { + let complete = false + function* pages() { + for (let page = 0; page < 100 && !complete; page++) yield page + } + await runInSeries(pages(), async () => { + complete = (await copy(k, 1000000)).complete + }) + if (!complete) throw new Error('Bootstrap incomplete') +} + +async function child() { + process.once('disconnect', () => process.exit(1)) + const childDeadline = setTimeout(() => process.exit(1), 20000) + childDeadline.unref() + + const filename = process.argv[3], + boundary = process.argv[4], + k = open(filename) + const park = phase => { + if (phase === boundary) { + writeFileSync(filename + '.marker', phase) + process.kill(process.pid, 'SIGKILL') + } + } + k.on('query-response', (_value, q) => { + const sql = q.sql.toLowerCase() + if (sql.startsWith('create trigger snapshot_journal_physical_12_delete')) park('install-after-ddl') + if (sql.startsWith('insert into `snapshot_journal_generation`')) park('install-after-generation') + if (sql.startsWith('insert into `snapshot_journal_bootstrap`')) park('install-after-bootstrap') + if (sql.startsWith('insert into `snapshot_journal_retention`')) park('install-after-retention') + if (sql.startsWith('update `snapshot_journal_generation`')) park('complete-after-state') + if (sql.startsWith('update `snapshot_journal_bootstrap` set `rowlimit`')) park('bootstrap-after-budget-bind') + if (sql.startsWith('insert into `snapshot_journal_physical`')) park('bootstrap-after-metadata') + if (sql.startsWith('update `snapshot_journal_bootstrap` set `cursor`')) park('bootstrap-after-progress') + }) + k.on('query', q => { + if (q.sql.toLowerCase().startsWith('update `snapshot_journal_generation`')) park('complete-before-state') + }) + try { + if (boundary.startsWith('install-')) { + await install(k, '1000000', receiptPolicy) + park('install-after-commit') + } else if (boundary.startsWith('bootstrap-')) { + await copy(k, 1000000) + park('bootstrap-after-commit') + } else { + await complete(k, receiptPolicy) + park('complete-after-commit') + } + throw new Error('Boundary not reached') + } finally { + await k.destroy() + } +} +async function killAt(filename, boundary) { + const processChild = fork(__filename, ['child', filename, boundary], { + stdio: ['ignore', 'ignore', 'pipe', 'ipc'] + }) + let stderr = '' + processChild.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-6000) + }) + const timeout = setTimeout(() => processChild.kill('SIGKILL'), 15000) + const result = await new Promise((resolve, reject) => { + processChild.once('error', reject) + processChild.once('exit', (code, signal) => resolve({ code, signal })) + }) + clearTimeout(timeout) + assert.equal(result.signal, 'SIGKILL', stderr) + assert.equal(readFileSync(filename + '.marker', 'utf8'), boundary) + return result +} +async function main() { + const directory = await mkdtemp(join(tmpdir(), 'ts569-journal-generation-kill-')), + results = [] + try { + await runInSeries( + [ + 'install-after-ddl', + 'install-after-generation', + 'install-after-bootstrap', + 'install-after-retention', + 'install-after-commit', + 'bootstrap-after-budget-bind', + 'bootstrap-after-metadata', + 'bootstrap-after-progress', + 'bootstrap-after-commit', + 'complete-before-state', + 'complete-after-state', + 'complete-after-commit' + ], + async boundary => { + const filename = join(directory, boundary + '.sqlite'), + k = open(filename), + source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: k }) + try { + await k.raw('PRAGMA journal_mode=WAL') + await source.migrate('journal generation process-loss fixture', 'synthetic-source') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)), + { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await seedArchiveClosure(source, user.userId, other.userId) + const original = {} + await runInSeries(tables, async table => { + original[table] = await k(table) + }) + if (boundary.startsWith('bootstrap-')) await install(k, '1000000', receiptPolicy) + if (boundary.startsWith('complete-')) { + await install(k, '1000000', receiptPolicy) + await finish(k) + } + const killed = await killAt(filename, boundary) + await k.transaction(async t => { + await t('snapshot_index_generation_v2') + .where('id', 0) + .update({ complete: t.ref('complete') }) + }) + const objects = await k('sqlite_master').whereRaw('lower(substr(name,1,17))=?', ['snapshot_journal_']) + const committed = boundary.endsWith('after-commit') + if (boundary.startsWith('install-')) assert.equal(objects.length, committed ? 61 : 0) + else if (boundary.startsWith('bootstrap-')) { + assert.equal((await read(k, receiptPolicy)).complete, false) + const progress = await k('snapshot_journal_bootstrap').first() + assert.equal(progress.rowsUsed, committed ? original.transactions.length : 0) + assert.equal(progress.rowLimit, committed ? 1000000 : null) + assert.equal( + progress.cursor, + committed ? JSON.stringify([Math.max(...original.transactions.map(row => row.transactionId))]) : null + ) + } else assert.equal((await read(k, receiptPolicy)).complete, committed) + await install(k, '1000000', receiptPolicy) + await finish(k) + await complete(k, receiptPolicy) + assert.equal((await read(k, receiptPolicy)).complete, true) + await exact(k) + let charged = 0 + await runInSeries( + [ + ...tables, + 'snapshot_profile_keys_v2', + 'snapshot_relation_keys_v2', + 'snapshot_certificate_field_keys_v2', + 'snapshot_global_keys_v2' + ], + async table => { + charged += Number((await k(table).count('* AS n').first()).n) + } + ) + assert.equal((await k('snapshot_journal_bootstrap').first()).rowsUsed, charged) + await runInSeries(tables, async table => { + assert.deepEqual(await k(table), original[table]) + }) + assert.deepEqual(await k.raw('PRAGMA foreign_key_check'), []) + results.push({ + boundary, + signal: killed.signal, + atomicity: committed ? 'committed' : 'rolled back', + writerLockReleased: true, + sourcePreserved: true, + resumedComplete: true + }) + } finally { + await source.destroy() + } + } + ) + await require('./snapshotJournalCaptureSqlite.cjs')() + await require('./snapshotJournalRetentionSqlite.cjs')() + await require('./snapshotJournalMaintenanceFixture.cjs')('WAL') + console.log( + JSON.stringify({ + status: 'SQLite journal generation native WAL process-loss checks', + results, + readerAdvertised: false, + limitations: [ + 'not yet a registered forward migration', + 'no MySQL implicit-DDL lifecycle qualification', + 'runtime quotas/registered lifecycle/receiver and platform-scale acceptance remain open' + ] + }) + ) + } finally { + await rm(directory, { recursive: true, force: true }) + } +} +if (process.argv[2] === 'child') + child().catch(error => { + console.error(error) + process.exitCode = 1 + }) +else + main().catch(error => { + console.error(error) + process.exitCode = 1 + }) diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotMysqlFixtureGroups.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotMysqlFixtureGroups.cjs new file mode 100644 index 000000000..05d214bb7 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotMysqlFixtureGroups.cjs @@ -0,0 +1,13 @@ +// Every native family remains mandatory; each child retains the 60-second bound. +const mysqlFixtureGroups = Object.freeze([ + 'archive', + 'profile', + 'relation', + 'certificate', + 'global-crash', + 'global-locks', + 'global-schedules', + 'global-seeks', + 'global-integration' +]) +module.exports = { mysqlFixtureGroups } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotProfileIndexCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotProfileIndexCrash.cjs new file mode 100644 index 000000000..8292df49d --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotProfileIndexCrash.cjs @@ -0,0 +1,239 @@ +const { migrateBeforeSqliteGeneration } = require('./snapshotHistoricalMigrations.cjs') +// Synthetic process-loss qualification, invoked by the existing native fixtures. +const assert = require('node:assert/strict') +const { spawn } = require('node:child_process') +const { randomUUID } = require('node:crypto') +const fs = require('node:fs/promises') +const { writeFileSync } = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { KnexMigrations } = require('../../out/src/storage/schema/KnexMigrations.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + readSnapshotProfileIndexState, + snapshotProfileTables, + SNAPSHOT_PROFILE_INDEX_MIGRATION +} = require('../../out/src/storage/schema/snapshotProfileIndexMigration.js') + +const phases = ['after-key-table', 'partial-triggers', 'before-cursor', 'after-cursor', 'after-commit'] +const migrationName = 'synthetic profile crash' +const migrationIdentity = 'synthetic-profile-crash' +const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } +const provider = options => + new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: knex(options) }) + +async function seed(options) { + const source = provider(options) + try { + if (options.client === 'better-sqlite3') await source.knex.raw('PRAGMA journal_mode = WAL') + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + await source.makeAvailable() + const migrationSource = new KnexMigrations('test', migrationName, migrationIdentity, 1024) + await source.knex.migrate.down({ + migrationSource, + name: SNAPSHOT_PROFILE_INDEX_MIGRATION, + disableTransactions: false + }) + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await runInSeries([0, 1, 2], async batch => { + await source.knex('tx_labels').insert( + Array.from({ length: 200 }, (_, index) => ({ + ...dates, + userId: user.userId, + label: `crash-label-${batch * 200 + index}`, + isDeleted: false + })) + ) + }) + await source.knex('tx_labels').where('txLabelId', 9).delete() + return { userId: user.userId, otherId: other.userId } + } finally { + await source.destroy() + } +} + +async function child(options, phase, marker) { + assert(phases.includes(phase)) + assert.equal(typeof process.send, 'function', 'Child requires its fixture parent') + const source = provider(options) + let cursorWritten = false + const cursor = query => + query.sql.startsWith('update `snapshot_profile_index_progress`') && + query.bindings[0] === 257 && + query.bindings.at(-1) === 3 + const park = event => { + writeFileSync(marker, JSON.stringify({ phase, event }), { mode: 0o600 }) + process.kill(process.pid, 'SIGKILL') + } + source.knex.on('query', query => { + if (phase === 'before-cursor' && cursor(query)) park('query') + if (phase === 'after-commit' && cursorWritten && query.sql.toLowerCase().startsWith('begin')) + park('next-transaction') + }) + source.knex.on('query-response', (_result, query) => { + const sql = query.sql.toLowerCase() + if (phase === 'after-key-table' && sql.startsWith('create table `snapshot_profile_keys`')) park('query-response') + if (phase === 'partial-triggers' && sql.startsWith('create trigger snapshot_profile_0_update ')) + park('query-response') + if (cursor(query)) { + cursorWritten = true + if (phase === 'after-cursor') park('query-response') + } + if (phase === 'after-commit' && cursorWritten && sql.startsWith('commit')) park('query-response') + }) + try { + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + throw new Error('Expected migration boundary was not reached') + } finally { + await source.destroy() + } +} + +async function terminateAt(options, phase) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-profile-boundary-')) + const marker = path.join(directory, 'boundary.json') + const processHandle = spawn(process.execPath, [__filename, 'child'], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + let stderr = '' + processHandle.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-65536) + }) + const exited = new Promise((resolve, reject) => { + processHandle.once('exit', (code, signal) => resolve({ code, signal })) + processHandle.once('error', reject) + }) + const timer = setTimeout(() => processHandle.kill('SIGKILL'), 15000) + try { + processHandle.send({ options, phase, marker }) + const result = await exited + assert.equal(result.signal, 'SIGKILL', stderr) + const observed = JSON.parse(await fs.readFile(marker, 'utf8')) + assert.equal(observed.phase, phase) + return { phase, event: observed.event, signal: result.signal } + } finally { + clearTimeout(timer) + if (processHandle.exitCode === null && processHandle.signalCode === null) { + processHandle.kill('SIGKILL') + await exited + } + await fs.rm(directory, { recursive: true, force: true }) + } +} + +async function qualify(options, phase) { + const { userId, otherId } = await seed(options) + const outcome = await terminateAt(options, phase) + const source = provider(options) + const database = source.knex + try { + assert.equal(await readSnapshotProfileIndexState(database), false) + if (['before-cursor', 'after-cursor', 'after-commit'].includes(phase)) { + const committed = phase === 'after-commit' + const progress = await database('snapshot_profile_index_progress').where('snapshotTableId', 3).first() + assert.equal(progress.afterRowId, committed ? 257 : 0) + assert.equal( + Number( + (await database('snapshot_profile_keys').where('snapshotTableId', 3).count({ count: '*' }).first()).count + ), + committed ? 256 : 0 + ) + } + await database('tx_labels').where('txLabelId', 3).update({ userId: otherId }) + await database('tx_labels').insert({ + ...dates, + txLabelId: 9, + userId, + label: 'reinserted-behind-cursor', + isDeleted: false + }) + // A killed migrator leaves Knex's lock claimed. This is fixture-owned recovery; + // the verified child is gone and no other migrator can own this isolated store. + await database.migrate.forceFreeMigrationsLock() + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + assert.equal(await readSnapshotProfileIndexState(database), true) + await runInSeries(snapshotProfileTables.entries(), async ([tableId, { table, key }]) => { + const expected = (await database(table).select('userId', key).orderBy(key)).map(row => ({ + snapshotUserId: row.userId, + snapshotRowId: row[key] + })) + assert.deepEqual( + await database('snapshot_profile_keys') + .where('snapshotTableId', tableId) + .select('snapshotUserId', 'snapshotRowId') + .orderBy('snapshotRowId'), + expected + ) + }) + assert.equal( + Number( + ( + await database('knex_migrations') + .where('name', SNAPSHOT_PROFILE_INDEX_MIGRATION) + .count({ count: '*' }) + .first() + ).count + ), + 1 + ) + return { ...outcome, journalPublishedAfterRecovery: true, independentProfileChangeAndLowIdInsert: true } + } finally { + await source.destroy() + } +} + +async function qualifySQLiteProfileIndexProcessLoss() { + const results = [] + await runInSeries(phases, async phase => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-profile-index-crash-')) + try { + results.push( + await qualify( + { + client: 'better-sqlite3', + connection: { filename: path.join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }, + phase + ) + ) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + return results +} + +async function qualifyMysqlProfileIndexProcessLoss(control, connection) { + // The calling fixture has already verified its disposable, pinned container. + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const results = [] + await runInSeries(phases, async phase => { + const database = 'ts569_profile_' + randomUUID().replaceAll('-', '') + await control.raw('CREATE DATABASE ??', [database]) + try { + results.push( + await qualify({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }, phase) + ) + } finally { + await control.raw('DROP DATABASE ??', [database]) + } + }) + return results +} + +if (process.argv[2] === 'child') { + assert.equal(typeof process.send, 'function', 'Child execution requires its fixture parent') + process.once('message', ({ options, phase, marker }) => { + child(options, phase, marker).catch(error => { + console.error(error) + process.exitCode = 1 + process.disconnect() + }) + }) +} +module.exports = { qualifySQLiteProfileIndexProcessLoss, qualifyMysqlProfileIndexProcessLoss } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotProfileIndexMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotProfileIndexMysql.cjs new file mode 100644 index 000000000..57402fd26 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotProfileIndexMysql.cjs @@ -0,0 +1,157 @@ +// Invoked only after the existing launcher verifies its disposable MySQL fixture. +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex } = require('knex') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { addSnapshotProfileIndexes } = require('../../out/src/storage/schema/snapshotProfileIndexMigration.js') + +async function waitFor(check) { + let complete = false + function* pendingAttempts() { + for (let attempt = 0; attempt < 1000 && !complete; attempt++) yield undefined + } + await runInSeries(pendingAttempts(), async () => { + complete = await check() + if (!complete) await new Promise(resolve => setTimeout(resolve, 5)) + }) + if (!complete) throw new Error('Synthetic MySQL lock observation deadline') +} + +async function qualifyMysqlProfileIndexLocks(control, connection) { + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const database = 'ts569_profile_' + randomUUID().replaceAll('-', '') + await control.raw('CREATE DATABASE ??', [database]) + const open = () => knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + const source = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: open() }) + const writer = open() + const observer = open() + const k = source.knex + let transaction + try { + await source.migrate('synthetic profile locks', 'synthetic-profile-locks') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await k('tx_labels').insert( + Array.from({ length: 20 }, (_, index) => ({ + userId: user.userId, + label: `lock-label-${index}`, + isDeleted: false, + created_at: new Date('2026-01-01'), + updated_at: new Date('2026-01-01') + })) + ) + const reset = async () => { + await k('snapshot_profile_keys').where('snapshotTableId', 3).delete() + await k('snapshot_profile_index_progress').where('snapshotTableId', 3).update({ afterRowId: 0, complete: false }) + } + const verify = async () => { + const expected = (await k('tx_labels').select('userId', 'txLabelId').orderBy('txLabelId')).map(row => ({ + snapshotUserId: row.userId, + snapshotRowId: row.txLabelId + })) + assert.deepEqual( + await k('snapshot_profile_keys') + .where('snapshotTableId', 3) + .select('snapshotUserId', 'snapshotRowId') + .orderBy('snapshotRowId'), + expected + ) + } + const waiting = async () => { + const [rows] = await observer.raw( + "SELECT COUNT(*) AS pending FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE WHERE l.OBJECT_SCHEMA=DATABASE() AND l.OBJECT_NAME='tx_labels'" + ) + return Number(rows[0].pending) > 0 + } + await reset() + transaction = await writer.transaction() + await transaction('tx_labels').where('txLabelId', 1).update({ userId: other.userId }) + let finished = false + const first = addSnapshotProfileIndexes(k).then( + () => { + finished = true + return { ok: true } + }, + error => ({ ok: false, error }) + ) + try { + await waitFor(waiting) + assert.equal(finished, false) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + const firstResult = await first + if (!firstResult.ok) throw firstResult.error + await verify() + + await reset() + let reached = false + let release + const gate = new Promise(resolve => { + release = resolve + }) + const prototype = Object.getPrototypeOf(k.client) + const original = prototype.query + const hadOwn = Object.hasOwn(prototype, 'query') + let intercepted = false + prototype.query = async function (conn, query) { + const sql = typeof query === 'string' ? query : query.sql + if (!intercepted && sql.startsWith('insert ignore into `snapshot_profile_keys`')) { + intercepted = true + reached = true + await gate + } + return await original.call(this, conn, query) + } + const second = addSnapshotProfileIndexes(k).then( + () => ({ ok: true }), + error => ({ ok: false, error }) + ) + let change + let secondResult + let changedResult + try { + await waitFor(async () => reached) + let writerDone = false + change = writer('tx_labels') + .where('txLabelId', 3) + .update({ userId: other.userId }) + .then( + () => { + writerDone = true + return { ok: true } + }, + error => ({ ok: false, error }) + ) + await waitFor(waiting) + assert.equal(writerDone, false) + } finally { + release() + if (hadOwn) prototype.query = original + else delete prototype.query + secondResult = await second + if (change !== undefined) changedResult = await change + } + if (!secondResult.ok) throw secondResult.error + if (changedResult !== undefined && !changedResult.ok) throw changedResult.error + await verify() + return { + bootstrapWaitsForCommittedOwner: true, + sourceLocksRetainedThroughKeyAndCursorCommit: true, + independentWriterMaintainsExactOwnerAfterCommit: true, + lockEvidence: 'performance_schema.data_lock_waits in the isolated fixture database' + } + } finally { + if (transaction !== undefined && !transaction.isCompleted()) await transaction.rollback() + await observer.destroy() + await writer.destroy() + await source.destroy() + await control.raw('DROP DATABASE ??', [database]) + } +} +module.exports = { qualifyMysqlProfileIndexLocks } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexCrash.cjs new file mode 100644 index 000000000..c8b4d7a41 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexCrash.cjs @@ -0,0 +1,325 @@ +const { migrateBeforeSqliteGeneration } = require('./snapshotHistoricalMigrations.cjs') +// Synthetic process-loss qualification, invoked by the existing native fixtures. +const assert = require('node:assert/strict') +const { spawn } = require('node:child_process') +const { randomUUID } = require('node:crypto') +const fs = require('node:fs/promises') +const { writeFileSync } = require('node:fs') +const os = require('node:os') +const path = require('node:path') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { KnexMigrations } = require('../../out/src/storage/schema/KnexMigrations.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + readSnapshotRelationIndexState, + snapshotNumericRelations, + SNAPSHOT_RELATION_INDEX_MIGRATION +} = require('../../out/src/storage/schema/snapshotRelationIndexMigration.js') + +const phases = [ + 'after-key-table', + 'after-first-index', + 'partial-observers', + 'partial-producers', + 'before-cursor', + 'after-cursor', + 'after-commit' +] +const migrationName = 'synthetic relation crash' +const migrationIdentity = 'synthetic-relation-crash' +const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } +const provider = options => + new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: knex(options) }) + +async function seed(options) { + const source = provider(options) + try { + if (options.client === 'better-sqlite3') await source.knex.raw('PRAGMA journal_mode = WAL') + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + await source.makeAvailable() + const migrationSource = new KnexMigrations('test', migrationName, migrationIdentity, 1024) + await source.knex.migrate.down({ + migrationSource, + name: SNAPSHOT_RELATION_INDEX_MIGRATION, + disableTransactions: false + }) + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await source + .knex('transactions') + .insert({ + ...dates, + transactionId: 1, + userId: user.userId, + status: 'completed', + reference: 'synthetic-relation', + isOutgoing: true, + satoshis: 0, + description: '' + }) + await source + .knex('outputs') + .insert({ + ...dates, + outputId: 1, + userId: user.userId, + transactionId: 1, + spendable: false, + change: false, + vout: 0, + satoshis: 1, + providedBy: 'you', + purpose: '', + type: 'P2PKH' + }) + await runInSeries(snapshotNumericRelations, async relation => { + await runInSeries([0, 1, 2], async batch => { + const rows = Array.from({ length: 200 }, (_, i) => batch * 200 + i + 1).filter(id => id !== 9) + const name = relation.left === 'tx_labels' ? 'label' : 'tag' + await source + .knex(relation.left) + .insert( + rows.map(id => ({ + ...dates, + [relation.leftKey]: id, + userId: user.userId, + [name]: `synthetic-${id}`, + isDeleted: false + })) + ) + await source + .knex(relation.table) + .insert( + rows.map(id => ({ ...dates, [relation.leftKey]: id, [relation.rightKey]: 1, isDeleted: id % 2 === 0 })) + ) + }) + }) + return { userId: user.userId, otherId: other.userId } + } finally { + await source.destroy() + } +} + +async function child(options, phase, marker) { + assert(phases.includes(phase)) + assert.equal(typeof process.send, 'function', 'Child requires its fixture parent') + const source = provider(options) + let cursorWritten = false + const cursor = query => + query.sql.startsWith('update `snapshot_relation_index_progress`') && + query.bindings[0] === 257 && + query.bindings[1] === 1 && + query.bindings.at(-1) === 0 + const park = event => { + writeFileSync(marker, JSON.stringify({ phase, event }), { mode: 0o600 }) + process.kill(process.pid, 'SIGKILL') + } + source.knex.on('query', query => { + if (phase === 'before-cursor' && cursor(query)) park('query') + if (phase === 'after-commit' && cursorWritten && query.sql.toLowerCase().startsWith('begin')) + park('next-transaction') + }) + source.knex.on('query-response', (_result, query) => { + const sql = query.sql.toLowerCase() + if (phase === 'after-key-table' && sql.startsWith('create table `snapshot_relation_keys`')) park('query-response') + if ( + phase === 'after-first-index' && + (sql.startsWith('create index `snapshot_relation_right`') || + sql.startsWith('alter table `snapshot_relation_keys` add index `snapshot_relation_right`')) + ) + park('query-response') + if (phase === 'partial-observers' && sql.startsWith('create trigger snapshot_relation_0_map_delete ')) + park('query-response') + if (phase === 'partial-producers' && sql.startsWith('create trigger snapshot_relation_0_map_insert ')) + park('query-response') + if (cursor(query)) { + cursorWritten = true + if (phase === 'after-cursor') park('query-response') + } + if (phase === 'after-commit' && cursorWritten && sql.startsWith('commit')) park('query-response') + }) + try { + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + throw new Error('Expected migration boundary was not reached') + } finally { + await source.destroy() + } +} + +async function terminateAt(options, phase) { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-relation-boundary-')) + const marker = path.join(directory, 'boundary.json') + const processHandle = spawn(process.execPath, [__filename, 'child'], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + let stderr = '' + processHandle.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-65536) + }) + const exited = new Promise((resolve, reject) => { + processHandle.once('exit', (code, signal) => resolve({ code, signal })) + processHandle.once('error', reject) + }) + const timer = setTimeout(() => processHandle.kill('SIGKILL'), 15000) + try { + processHandle.send({ options, phase, marker }) + const result = await exited + assert.equal(result.signal, 'SIGKILL', stderr) + const observed = JSON.parse(await fs.readFile(marker, 'utf8')) + assert.equal(observed.phase, phase) + return { phase, event: observed.event, signal: result.signal } + } finally { + clearTimeout(timer) + if (processHandle.exitCode === null && processHandle.signalCode === null) { + processHandle.kill('SIGKILL') + await exited + } + await fs.rm(directory, { recursive: true, force: true }) + } +} + +async function qualify(options, phase) { + const { userId, otherId } = await seed(options) + const outcome = await terminateAt(options, phase) + const source = provider(options) + const database = source.knex + try { + assert.equal(await readSnapshotRelationIndexState(database), false) + if (['before-cursor', 'after-cursor', 'after-commit'].includes(phase)) { + const committed = phase === 'after-commit' + const progress = await database('snapshot_relation_index_progress').where('snapshotTableId', 0).first() + assert.equal(progress.afterLeftId, committed ? 257 : 0) + assert.equal(progress.afterRightId, committed ? 1 : 0) + assert.equal( + Number( + (await database('snapshot_relation_keys').where('snapshotTableId', 0).count({ count: '*' }).first()).count + ), + committed ? 256 : 0 + ) + } + await runInSeries(snapshotNumericRelations, async relation => { + await database(relation.left).where(relation.leftKey, 3).update({ userId: otherId }) + const name = relation.left === 'tx_labels' ? 'label' : 'tag' + await database(relation.left).insert({ + ...dates, + [relation.leftKey]: 9, + userId, + [name]: 'reinserted-behind-cursor', + isDeleted: false + }) + await database(relation.table).insert({ + ...dates, + [relation.leftKey]: 9, + [relation.rightKey]: 1, + isDeleted: true + }) + await database(relation.table).where(relation.leftKey, 4).delete() + }) + // A killed migrator leaves Knex's lock claimed. This is fixture-owned recovery; + // the verified child is gone and no other migrator can own this isolated store. + await database.migrate.forceFreeMigrationsLock() + await migrateBeforeSqliteGeneration(source, migrationName, migrationIdentity) + assert.equal(await readSnapshotRelationIndexState(database), true) + await runInSeries(snapshotNumericRelations.entries(), async ([tableId, relation]) => { + const left = new Map((await database(relation.left)).map(row => [row[relation.leftKey], row.userId])) + const right = new Map((await database(relation.right)).map(row => [row[relation.rightKey], row.userId])) + const expected = [] + for (const row of await database(relation.table)) { + const owners = new Map() + for (const [userId, bit] of [ + [left.get(row[relation.leftKey]), 1], + [right.get(row[relation.rightKey]), 2] + ]) + if (userId !== undefined) owners.set(userId, (owners.get(userId) ?? 0) | bit) + for (const [userId, membership] of owners) + expected.push({ + snapshotTableId: tableId, + snapshotUserId: userId, + snapshotLeftId: row[relation.leftKey], + snapshotRightId: row[relation.rightKey], + snapshotMembership: membership + }) + } + expected.sort( + (a, b) => + a.snapshotUserId - b.snapshotUserId || + a.snapshotLeftId - b.snapshotLeftId || + a.snapshotRightId - b.snapshotRightId + ) + assert.deepEqual( + await database('snapshot_relation_keys') + .where('snapshotTableId', tableId) + .orderBy(['snapshotUserId', 'snapshotLeftId', 'snapshotRightId']), + expected + ) + }) + assert.equal( + Number( + ( + await database('knex_migrations') + .where('name', SNAPSHOT_RELATION_INDEX_MIGRATION) + .count({ count: '*' }) + .first() + ).count + ), + 1 + ) + return { ...outcome, journalPublishedAfterRecovery: true, independentProfileChangeAndLowIdInsert: true } + } finally { + await source.destroy() + } +} + +async function qualifySQLiteRelationIndexProcessLoss() { + const results = [] + await runInSeries(phases, async phase => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-relation-index-crash-')) + try { + results.push( + await qualify( + { + client: 'better-sqlite3', + connection: { filename: path.join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }, + phase + ) + ) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + return results +} + +async function qualifyMysqlRelationIndexProcessLoss(control, connection) { + // The calling fixture has already verified its disposable, pinned container. + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const results = [] + await runInSeries(phases, async phase => { + const database = 'ts569_relation_' + randomUUID().replaceAll('-', '') + await control.raw('CREATE DATABASE ??', [database]) + try { + results.push( + await qualify({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }, phase) + ) + } finally { + await control.raw('DROP DATABASE ??', [database]) + } + }) + return results +} + +if (process.argv[2] === 'child') { + assert.equal(typeof process.send, 'function', 'Child execution requires its fixture parent') + process.once('message', ({ options, phase, marker }) => { + child(options, phase, marker).catch(error => { + console.error(error) + process.exitCode = 1 + process.disconnect() + }) + }) +} +module.exports = { qualifySQLiteRelationIndexProcessLoss, qualifyMysqlRelationIndexProcessLoss } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexMysql.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexMysql.cjs new file mode 100644 index 000000000..a3f055ec8 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexMysql.cjs @@ -0,0 +1,259 @@ +// Invoked only by the verified disposable native MySQL fixture. +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex } = require('knex') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + snapshotNumericRelations: relations, + addSnapshotRelationIndexes: install +} = require('../../out/src/storage/schema/snapshotRelationIndexMigration.js') +const settle = promise => + promise.then( + value => ({ ok: true, value }), + error => ({ ok: false, error }) + ) +function success(result) { + if (!result.ok) throw result.error +} +async function until(check) { + let result + function* attempts() { + for (let i = 0; i < 1000 && !result; i++) yield i + } + await runInSeries(attempts(), async () => { + result = await check() + if (!result) await new Promise(resolve => setTimeout(resolve, 5)) + }) + if (!result) throw new Error('Native lock observation exceeded five seconds') + return result +} + +async function fixture(admin, connection, isolation, relation, tableId) { + const database = 'ts569_relation_locks_' + randomUUID().replaceAll('-', '') + await admin.raw('CREATE DATABASE ??', [database]) + const open = () => knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + const k = open(), + writer = open(), + observer = open() + const events = [] + let transaction + try { + await runInSeries([k, writer, observer], async client => { + await client.raw('SET SESSION TRANSACTION ISOLATION LEVEL ' + isolation.toUpperCase()) + }) + await runInSeries(relations, async p => { + await runInSeries( + [ + [p.left, p.leftKey], + [p.right, p.rightKey] + ], + async ([table, key]) => { + await k.schema.createTable(table, t => { + t.integer(key).primary() + t.integer('userId').notNullable() + }) + await k(table).insert({ [key]: 1, userId: 1 }) + } + ) + await k.schema.createTable(p.table, t => { + t.integer(p.leftKey) + t.integer(p.rightKey) + t.boolean('isDeleted') + t.primary([p.leftKey, p.rightKey]) + t.index(p.rightKey) + }) + }) + await install(k) + const p = relation + const mapping = { [p.leftKey]: 1, [p.rightKey]: 1, isDeleted: false } + const expected = () => [ + { snapshotTableId: tableId, snapshotUserId: 1, snapshotLeftId: 1, snapshotRightId: 1, snapshotMembership: 2 }, + { snapshotTableId: tableId, snapshotUserId: 2, snapshotLeftId: 1, snapshotRightId: 1, snapshotMembership: 1 } + ] + const verify = async () => + assert.deepEqual( + await k('snapshot_relation_keys').where('snapshotTableId', tableId).orderBy('snapshotUserId'), + expected() + ) + const reset = async () => { + await k(p.table).delete() + await k(p.left).where(p.leftKey, 1).update({ userId: 1 }) + } + const waiting = async (table, requester) => { + const [rows] = await observer.raw( + 'SELECT l.OBJECT_NAME AS tableName, l.LOCK_MODE AS lockMode, t.PROCESSLIST_ID AS requester FROM performance_schema.data_lock_waits w JOIN performance_schema.data_locks l ON l.ENGINE_LOCK_ID=w.REQUESTING_ENGINE_LOCK_ID AND l.ENGINE=w.ENGINE JOIN performance_schema.threads t ON t.THREAD_ID=l.THREAD_ID WHERE l.OBJECT_SCHEMA=DATABASE() AND l.OBJECT_NAME=? AND t.PROCESSLIST_ID=?', + [table, requester] + ) + return rows.length ? rows : false + } + const connectionId = async client => Number((await client.raw('SELECT CONNECTION_ID() AS id'))[0][0].id) + const kId = await connectionId(k) + const writerId = await connectionId(writer) + + // The map cannot use an owner whose transaction has not committed yet. + transaction = await writer.transaction() + await transaction(p.left).where(p.leftKey, 1).update({ userId: 2 }) + let done = false + const first = settle( + k(p.table) + .insert(mapping) + .then(() => { + done = true + }) + ) + try { + // RR's parent producer protects the empty map range with a next-key lock, + // so the concurrent insert waits there before reaching its parent lookup. + const blockedTable = isolation === 'repeatable read' ? p.table : p.left + const locks = await until(() => waiting(blockedTable, kId)) + assert.ok( + locks.every(lock => + isolation === 'repeatable read' ? lock.lockMode.includes('INSERT_INTENTION') : lock.lockMode.startsWith('S') + ) + ) + assert.equal(done, false) + events.push({ case: 'map waits for committed parent owner', locks }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + success(await first) + await verify() + + // The map producer retains parent protection until its own commit. + await reset() + transaction = await writer.transaction() + await transaction(p.table).insert(mapping) + done = false + const second = settle( + k(p.left) + .where(p.leftKey, 1) + .update({ userId: 2 }) + .then(() => { + done = true + }) + ) + try { + const locks = await until(() => waiting(p.left, kId)) + assert.equal(done, false) + events.push({ case: 'parent waits for map publication', locks }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + success(await second) + await verify() + + await reset() + transaction = await k.transaction() + try { + assert.equal((await transaction(p.left).where(p.leftKey, 1).first()).userId, 1) + await writer(p.left).where(p.leftKey, 1).update({ userId: 2 }) + await transaction(p.table).insert(mapping) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + await verify() + events.push({ case: 'map producer ignores earlier consistent owner snapshot' }) + + await reset() + transaction = await k.transaction() + try { + assert.equal((await transaction(p.table)).length, 0) + await writer(p.table).insert(mapping) + await transaction(p.left).where(p.leftKey, 1).update({ userId: 2 }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + await verify() + events.push({ case: 'parent producer ignores earlier consistent map snapshot' }) + + // Replay a bounded bootstrap while an independent owner change is pending. + await k('snapshot_relation_keys').where('snapshotTableId', tableId).delete() + await k('snapshot_relation_index_progress') + .where('snapshotTableId', tableId) + .update({ afterLeftId: 0, afterRightId: 0, complete: false }) + transaction = await writer.transaction() + await transaction(p.left).where(p.leftKey, 1).update({ userId: 1 }) + await transaction(p.left).where(p.leftKey, 1).update({ userId: 2 }) + const bootstrapWait = settle(install(k)) + try { + const locks = await until(() => waiting(p.table, kId)) + events.push({ case: 'bootstrap waits for current committed relation', locks }) + await transaction.commit() + } finally { + if (!transaction.isCompleted()) await transaction.rollback() + } + success(await bootstrapWait) + await verify() + + // Pause after both source/parent reads but before auxiliary key publication. + await k(p.left).where(p.leftKey, 1).update({ userId: 1 }) + await k('snapshot_relation_keys').where('snapshotTableId', tableId).delete() + await k('snapshot_relation_index_progress') + .where('snapshotTableId', tableId) + .update({ afterLeftId: 0, afterRightId: 0, complete: false }) + let reached = false, + release + const gate = new Promise(resolve => { + release = resolve + }) + const prototype = Object.getPrototypeOf(k.client) + const original = prototype.query + const hadOwn = Object.hasOwn(prototype, 'query') + prototype.query = async function (conn, query) { + const sql = typeof query === 'string' ? query : query.sql + if (!reached && sql.startsWith('insert into `snapshot_relation_keys`')) { + reached = true + await gate + } + return await original.call(this, conn, query) + } + const bootstrap = settle(install(k)) + let update + try { + await until(async () => reached) + done = false + update = settle( + writer(p.left) + .where(p.leftKey, 1) + .update({ userId: 2 }) + .then(() => { + done = true + }) + ) + const locks = await until(() => waiting(p.left, writerId)) + assert.equal(done, false) + events.push({ case: 'bootstrap retains source protection through key and cursor commit', locks }) + } finally { + release() + if (hadOwn) prototype.query = original + else delete prototype.query + success(await bootstrap) + if (update !== undefined) success(await update) + } + await verify() + return { isolation, relation: p.table, events } + } finally { + if (transaction !== undefined && !transaction.isCompleted()) await transaction.rollback() + await observer.destroy() + await writer.destroy() + await k.destroy() + await admin.raw('DROP DATABASE ??', [database]) + } +} + +async function qualifyMysqlRelationIndexLocks(control, connection) { + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const results = [] + await runInSeries(['read committed', 'repeatable read'], async isolation => { + await runInSeries(relations.entries(), async ([tableId, relation]) => { + results.push(await fixture(control, connection, isolation, relation, tableId)) + }) + }) + return results +} +module.exports = { qualifyMysqlRelationIndexLocks } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexSeeks.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexSeeks.cjs new file mode 100644 index 000000000..16e30af11 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotRelationIndexSeeks.cjs @@ -0,0 +1,157 @@ +// Invoked only by the verified disposable native MySQL fixture. +const assert = require('node:assert/strict') +const { randomUUID } = require('node:crypto') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { + createKnexWalletSnapshotPageReader: pageReader +} = require('../../out/src/storage/snapshot/KnexWalletReadSnapshot.js') + +async function seedLargeRelations(k, userId, otherId) { + const date = new Date('2026-01-01T00:00:00Z') + const common = { created_at: date, updated_at: date, isDeleted: false } + const starts = Array.from({ length: 32 }, (_, index) => 100 + index * 256) + await runInSeries(starts, async start => { + const ids = Array.from({ length: Math.min(256, 8292 - start) }, (_, i) => start + i) + await k.transaction(async trx => { + await trx('tx_labels').insert( + ids.map(id => ({ ...common, txLabelId: id, userId: id % 2 ? otherId : userId, label: 'large-' + id })) + ) + await trx('output_tags').insert( + ids.map(id => ({ ...common, outputTagId: id, userId: id % 2 ? otherId : userId, tag: 'large-' + id })) + ) + await trx('tx_labels_map').insert(ids.map(id => ({ ...common, txLabelId: id, transactionId: id % 2 ? 2 : 1 }))) + await trx('output_tags_map').insert(ids.map(id => ({ ...common, outputTagId: id, outputId: id % 2 ? 2 : 1 }))) + }) + }) +} + +async function handlerCounters(source, view) { + return await view.read(async trx => + Object.fromEntries( + (await source.toDb(trx).raw("SHOW SESSION STATUS LIKE 'Handler_read_%'"))[0].map(row => [ + row.Variable_name, + Number(row.Value) + ]) + ) + ) +} + +async function observePage(source, view, read, table, leftId) { + const queries = [] + const listener = q => { + if (q.sql.startsWith('select') && q.sql.includes('snapshot_relation_keys') && q.sql.includes('cross join')) + queries.push(q) + } + const before = await handlerCounters(source, view) + source.knex.on('query', listener) + let page + try { + page = await read( + table, + { version: 1, snapshotId: 'large-seek-fixture', table, after: [leftId, 1] }, + { maxRows: 16, maxBytes: 131072 } + ) + } finally { + source.knex.off('query', listener) + } + const after = await handlerCounters(source, view) + const deltas = Object.fromEntries(Object.keys(after).map(key => [key, after[key] - before[key]])) + const expectedIds = Array.from({ length: Math.min(16, (8290 - leftId) / 2) }, (_, i) => leftId + (i + 1) * 2) + const field = table === 'txLabelMaps' ? 'txLabelId' : 'outputTagId' + assert.deepEqual( + page.rows.map(row => row[field]), + expectedIds + ) + const plans = [] + await runInSeries(queries, async q => { + plans.push(await view.read(async trx => (await source.toDb(trx).raw('EXPLAIN ' + q.sql, q.bindings))[0])) + }) + assert.ok(deltas.Handler_read_next <= 64, 'The page must not scan prior or foreign relation keys') + assert.ok(deltas.Handler_read_rnd_next <= 64, 'The page must not scan rows into a temporary table') + assert.ok( + plans.every(plan => plan[0].table === 'snapshot_relation_keys' && plan[0].type === 'range'), + 'Both page passes must seek the composite profile range' + ) + return { table, after: leftId, rows: page.rows.length, deltas, plans, queries } +} + +async function largeSeeks(source, userId, otherId) { + await seedLargeRelations(source.knex, userId, otherId) + const results = [] + // A statistics refresh can hide a full-prefix scan after bootstrap or bulk + // writes. Prove the same bounded native work on both sides of that boundary. + await runInSeries(['fresh', 'refreshed'], async statistics => { + if (statistics === 'refreshed') + await source.knex.raw('ANALYZE TABLE snapshot_relation_keys, tx_labels_map, output_tags_map') + const view = await source.openReadSnapshot() + const read = pageReader(source, userId, 'large-seek-fixture', view, true, true) + try { + await runInSeries(['txLabelMaps', 'outputTagMaps'], async table => { + await read(table, undefined, { maxRows: 1 }) + await runInSeries([1500, 7000, 8280], async leftId => { + results.push({ statistics, ...(await observePage(source, view, read, table, leftId)) }) + }) + }) + } finally { + await view.close() + } + }) + return results +} + +async function qualifyMysqlRelationIndexSeeks(control, connection) { + assert.equal(connection.host, '127.0.0.1') + assert.equal(connection.database, 'ts569_snapshot') + const database = 'ts569_relation_seeks_' + randomUUID().replaceAll('-', '') + await control.raw('CREATE DATABASE ??', [database]) + const source = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ client: 'mysql2', connection: { ...connection, database }, pool: { min: 1, max: 1 } }) + }) + try { + await source.migrate('synthetic relation seek', 'synthetic-relation-seek') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } + await runInSeries( + [ + [1, user.userId], + [2, other.userId] + ], + async ([id, userId]) => { + await source.knex('transactions').insert({ + ...dates, + transactionId: id, + userId, + status: 'completed', + reference: 'relation-' + id, + isOutgoing: true, + satoshis: 0, + description: '' + }) + await source.knex('outputs').insert({ + ...dates, + outputId: id, + userId, + transactionId: id, + spendable: false, + change: false, + vout: 0, + satoshis: 1, + providedBy: 'you', + purpose: '', + type: 'P2PKH' + }) + } + ) + return await largeSeeks(source, user.userId, other.userId) + } finally { + await source.destroy() + await control.raw('DROP DATABASE ??', [database]) + } +} +module.exports = { qualifyMysqlRelationIndexSeeks } diff --git a/packages/wallet/wallet-toolbox/test/storage/snapshotSqliteGenerationCrash.cjs b/packages/wallet/wallet-toolbox/test/storage/snapshotSqliteGenerationCrash.cjs new file mode 100644 index 000000000..3c22f4d96 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/storage/snapshotSqliteGenerationCrash.cjs @@ -0,0 +1,275 @@ +// Actual built-source, registered-migrator SQLite process-loss qualification. +const assert = require('node:assert/strict') +const { spawn } = require('node:child_process') +const fs = require('node:fs/promises') +const { writeFileSync } = require('node:fs') +const path = require('node:path') +const os = require('node:os') +const { knex } = require('knex') +const { StorageKnex } = require('../../out/src/storage/StorageKnex.js') +const { StorageProvider } = require('../../out/src/storage/StorageProvider.js') +const { runInSeries } = require('../../out/src/utility/runInSeries.js') +const { migration, readGenerationIndexState } = require('../../out/src/storage/schema/snapshotSqliteIndexState.js') +const { names, metadata, progress } = require('../../out/src/storage/schema/snapshotSqliteIndexGeneration.js') +const { retiredTables } = require('../../out/src/storage/schema/snapshotSqliteLegacyOwnership.js') +const { migrateBeforeSqliteGeneration } = require('./snapshotHistoricalMigrations.cjs') +const { oracle } = require('./snapshotGlobalIndexFixtures.cjs') +const phases = [ + 'install-lock', + 'install-first-table', + 'install-partial-observers', + 'install-before-commit', + 'install-committed', + 'copy-before-cursor', + 'copy-after-cursor', + 'copy-before-commit', + 'copy-committed', + 'complete-before-commit', + 'copy-completed', + 'retirement-before-commit', + 'retirement-page-committed', + 'retirement-before-drop', + 'retirement-completed', + 'publication-before-insert', + 'publication-committed' +] +const dates = { created_at: new Date('2026-01-01'), updated_at: new Date('2026-01-01') } +function provider(filename) { + return new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + knex: knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) +} +async function seed(filename) { + const source = provider(filename) + try { + await source.knex.raw('PRAGMA journal_mode=WAL') + await migrateBeforeSqliteGeneration(source, 'generation loss', 'synthetic-generation-loss') + await source.makeAvailable() + const { user } = await source.findOrInsertUser('02' + '11'.repeat(32)) + const { user: other } = await source.findOrInsertUser('03' + '22'.repeat(32)) + await source.knex('proven_txs').insert({ + ...dates, + provenTxId: 7, + txid: '7'.repeat(64), + height: 1, + index: 0, + merklePath: Buffer.from([1]), + rawTx: Buffer.from([1]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + await runInSeries([0, 100, 200, 300, 400, 500], async start => { + await source.knex('transactions').insert( + Array.from({ length: 100 }, (_, i) => ({ + ...dates, + transactionId: (start + i + 1) * 2, + userId: user.userId, + txid: 'a'.repeat(64), + provenTxId: 7, + status: 'completed', + reference: 'generation-' + (start + i), + isOutgoing: true, + satoshis: 0, + description: '' + })) + ) + }) + return { + userId: user.userId, + otherId: other.userId, + rows: await source.knex('transactions').orderBy('transactionId') + } + } finally { + await source.destroy() + } +} +async function child(filename, phase, marker) { + assert(phases.includes(phase)) + assert.equal(typeof process.send, 'function', 'Native fixture child requires its parent') + const source = provider(filename) + const k = source.knex + let stage = 'unstarted' + const park = observed => { + if (observed !== phase) return + writeFileSync(marker, JSON.stringify({ phase, stage }), { mode: 0o600 }) + process.kill(process.pid, 'SIGKILL') + } + let committing + const committed = () => { + const current = committing + committing = undefined + if (current === undefined) return + stage = 'idle' + if (current === 'install') park('install-committed') + if (current === 'copy') park('copy-committed') + if (current === 'complete') park('copy-completed') + if (current === 'retire') park('retirement-page-committed') + if (current === 'retire-complete') park('retirement-completed') + } + const firstCursor = q => q.sql.startsWith('update `snapshot_index_rebuild_v2`') && q.bindings.includes(512) + k.on('query', q => { + const sql = q.sql.toLowerCase() + // Knex does not emit query-response for every SQLite COMMIT. The next + // serialized query also proves that the prior transaction completed. + if (sql !== 'commit;') committed() + if (sql.startsWith('create table if not exists snapshot_index_install_lock_v2')) stage = 'install' + if (stage === 'install' && sql === 'commit;') park('install-before-commit') + if (firstCursor(q)) park('copy-before-cursor') + if (stage === 'copy' && sql === 'commit;') park('copy-before-commit') + if (stage === 'complete' && sql === 'commit;') park('complete-before-commit') + if (stage === 'retire' && sql === 'commit;') park('retirement-before-commit') + if (sql === 'commit;') committing = stage + if (sql.startsWith('drop table')) park('retirement-before-drop') + if (sql.startsWith('insert into `knex_migrations`') && q.bindings.includes(migration)) + park('publication-before-insert') + }) + k.on('query-response', (_value, q) => { + const sql = q.sql.toLowerCase() + if (sql.startsWith('create table if not exists snapshot_index_install_lock_v2')) park('install-lock') + if (sql.startsWith('create table "snapshot_profile_keys_v2"')) park('install-first-table') + if (sql.startsWith('create trigger "snapshot_identity_before_transactions_insert"')) + park('install-partial-observers') + if (firstCursor(q)) { + stage = 'copy' + park('copy-after-cursor') + } + if (sql.startsWith('update `snapshot_index_generation_v2` set `complete` = ?') && q.bindings[0] === true) + stage = 'complete' + if (sql.startsWith('delete from `snapshot_global_edges`')) stage = 'retire' + if ( + sql.startsWith('update `snapshot_index_generation_v2` set `retiretable` = ?') && + q.bindings[0] === retiredTables.length + ) + stage = 'retire-complete' + if (sql === 'commit;') committed() + if (sql.startsWith('insert into `knex_migrations`') && q.bindings.includes(migration)) park('publication-committed') + }) + try { + await source.migrate('generation loss', 'synthetic-generation-loss') + throw new Error('Requested generation boundary was not reached: ' + phase) + } finally { + await source.destroy() + } +} +async function terminateAt(filename, phase, directory) { + const marker = path.join(directory, 'boundary.json') + const handle = spawn(process.execPath, [__filename, 'child'], { stdio: ['ignore', 'ignore', 'pipe', 'ipc'] }) + let stderr = '' + handle.stderr.on('data', chunk => { + stderr = (stderr + chunk.toString()).slice(-65536) + }) + const exited = new Promise((resolve, reject) => { + handle.once('exit', (code, signal) => resolve({ code, signal })) + handle.once('error', reject) + }) + const timer = setTimeout(() => handle.kill('SIGKILL'), 15000) + try { + handle.send({ filename, phase, marker }) + const result = await exited + assert.equal(result.signal, 'SIGKILL', stderr) + assert.equal(JSON.parse(await fs.readFile(marker, 'utf8')).phase, phase) + return result + } finally { + clearTimeout(timer) + if (handle.exitCode === null && handle.signalCode === null) { + handle.kill('SIGKILL') + await exited + } + } +} +async function qualifySQLiteGenerationProcessLoss() { + const results = [] + await runInSeries(phases, async phase => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), 'ts569-sqlite-generation-')) + const filename = path.join(directory, 'wallet.sqlite') + let source + try { + const seeded = await seed(filename) + const result = await terminateAt(filename, phase, directory) + source = provider(filename) + const k = source.knex + assert.deepEqual(await k('transactions').orderBy('transactionId'), seeded.rows) + const early = [ + 'install-lock', + 'install-first-table', + 'install-partial-observers', + 'install-before-commit' + ].includes(phase) + assert.equal(await k.schema.hasTable(metadata), !early) + if (early) await oracle(k) + else { + assert.equal(await readGenerationIndexState(k), phase === 'publication-committed' ? 'v2' : false) + const after = (await k(progress).where('stream', 0).first()).afterId + if (['install-committed', 'copy-before-cursor', 'copy-after-cursor', 'copy-before-commit'].includes(phase)) + assert.equal(after, 0) + else if (phase === 'copy-committed') assert.equal(after, 512) + else assert.equal(after, 1200) + } + if (phase === 'retirement-before-commit') assert.equal((await k('snapshot_global_edges')).length, 600) + if (phase === 'retirement-page-committed') assert.equal((await k('snapshot_global_edges')).length, 344) + const published = await k('knex_migrations').where('name', migration) + assert.equal(published.length, phase === 'publication-committed' ? 1 : 0) + // Independent mutations below the committed source cursor must be represented after recovery. + await k('transactions').where('transactionId', 2).update({ userId: seeded.otherId }) + await k('transactions').where('transactionId', 4).delete() + await k('transactions').insert({ ...seeded.rows[0], transactionId: 3, reference: 'new-low-key' }) + // This exact owned child has exited; no other migrator can own the fixture lock. + await k.migrate.forceFreeMigrationsLock() + assert.equal(await source.migrate('generation loss', 'synthetic-generation-loss'), migration) + assert.equal(await readGenerationIndexState(k), 'v2') + const mapping = new Map([ + ['snapshot_global_edges', names.edges], + ['snapshot_global_keys', names.keys], + ['snapshot_global_guards', names.guards] + ]) + const rebuilt = new Proxy(k, { + apply(target, self, args) { + if (mapping.has(args[0])) args[0] = mapping.get(args[0]) + return Reflect.apply(target, self, args) + } + }) + await oracle(rebuilt) + const actual = await k(names.profile).where('snapshotTableId', 0).orderBy('snapshotRowId') + const expected = await k('transactions') + .select({ snapshotRowId: 'transactionId', snapshotUserId: 'userId' }) + .orderBy('transactionId') + assert.deepEqual( + actual.map(({ snapshotRowId, snapshotUserId }) => ({ snapshotRowId, snapshotUserId })), + expected + ) + await runInSeries(retiredTables, async table => { + assert.equal(await k.schema.hasTable(table), false) + }) + assert.equal((await k('knex_migrations').where('name', migration)).length, 1) + assert.equal((await k.raw('PRAGMA foreign_keys'))[0].foreign_keys, 1) + results.push({ + phase, + signal: result.signal, + sourceRowsRetained: true, + completeJournalAfterRecovery: true, + independentLowKeyWrites: true + }) + } finally { + if (source) await source.destroy() + await fs.rm(directory, { recursive: true, force: true }) + } + }) + return results +} +if (process.argv[2] === 'child') { + assert.equal(typeof process.send, 'function', 'Native fixture child requires its parent') + process.once('message', ({ filename, phase, marker }) => { + child(filename, phase, marker).catch(error => { + console.error(error) + process.exitCode = 1 + process.disconnect() + }) + }) +} +module.exports = { qualifySQLiteGenerationProcessLoss } diff --git a/packages/wallet/wallet-toolbox/test/utils/mysqlReadSnapshotFixture.ts b/packages/wallet/wallet-toolbox/test/utils/mysqlReadSnapshotFixture.ts new file mode 100644 index 000000000..cdddb6243 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/mysqlReadSnapshotFixture.ts @@ -0,0 +1,45 @@ +import { knex } from 'knex' +import { StorageKnex } from '../../src/storage/StorageKnex' +import { StorageProvider } from '../../src/storage/StorageProvider' + +export function mysqlReadSnapshotFixture(failAt?: string, selectedRows: readonly unknown[] = []) { + const db = knex({ client: 'mysql2' }) + const storage = new StorageKnex({ ...StorageProvider.createStorageBaseOptions('test'), knex: db }) + const events: string[] = [] + const failure = new Error('synthetic database failure') + const connection = { + destroy() { + events.push('CLOSE') + }, + query( + query: { sql: string }, + _bindings: unknown, + callback: (error: Error | null, rows?: unknown[], fields?: unknown[]) => void + ) { + const sql = query.sql.toUpperCase().replace(/;$/, '') + events.push(sql) + if (sql === failAt) return callback(failure) + // Exercise the actual installed Knex transaction builder against MySQL's + // comma-separated characteristic grammar. The combined options emitted + // invalid syntax before this regression was found with real MySQL 8.4. + if (sql.includes('REPEATABLE READ READ ONLY')) return callback(new Error('invalid MySQL syntax')) + callback(null, sql.startsWith('SELECT') ? structuredClone([...selectedRows]) : [], []) + } + } + const acquire = jest.spyOn(db.client, 'acquireConnection').mockResolvedValue(connection) + jest.spyOn(db.client, 'releaseConnection').mockImplementation(() => { + events.push('RELEASE') + return Promise.resolve() + }) + storage._settings = { + created_at: new Date(0), + updated_at: new Date(0), + storageIdentityKey: 'synthetic-storage', + storageName: 'synthetic MySQL', + chain: 'test', + dbtype: 'MySQL', + maxOutputScript: 1024 + } + jest.spyOn(storage, 'makeAvailable').mockResolvedValue(storage._settings) + return { storage, db, events, failure, acquire } +} diff --git a/packages/wallet/wallet-toolbox/test/utils/remoteSnapshotReaderFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/remoteSnapshotReaderFixtures.ts new file mode 100644 index 000000000..8288caa0a --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/remoteSnapshotReaderFixtures.ts @@ -0,0 +1,122 @@ +// Intended location: wallet-toolbox/test/utils/remoteSnapshotReaderFixtures.ts +import type { WalletSnapshotTable } from '../../src/storage/snapshot/WalletReadSnapshot' +import { + snapshotArchiveReaderRequestId, + type SnapshotArchiveReaderRequest +} from '../../src/storage/snapshot/archive/SnapshotArchiveReaderRequest' +import type { SnapshotArchiveRpcCall } from '../../src/storage/snapshot/archive/SnapshotArchiveTransport' +import { SnapshotArchiveTransport } from '../../src/storage/snapshot/archive/SnapshotArchiveTransport' +import { encodeSyncTransfer } from '../../src/storage/remoting/SyncTransfer' +import { fixture, expected, hash, rehash, tables } from './snapshotArchiveDirectoryFixtures' + +export function label(id: number, text = `label-${id}`) { + return { + created_at: new Date('2026-01-01T00:00:00.000Z'), + updated_at: new Date('2026-01-01T00:00:00.000Z'), + txLabelId: id, + userId: 7, + label: text, + isDeleted: id % 3 === 0 + } +} + +/** Independent complete receipt chain with real binary frames and no SQL/network dependency. */ +export function remoteReaderFixture(rows: object[], table: WalletSnapshotTable = 'txLabels', frameRows = 17) { + const { directory, binding } = fixture() + const frames: Uint8Array[] = [] + directory.receipts = [] + for (const name of tables) { + const selected = name === table ? rows : [] + const count = Math.max(1, Math.ceil(selected.length / frameRows)) + for (let index = 0; index < count; index++) { + const values = selected.slice(index * frameRows, (index + 1) * frameRows) + const bytes = encodeSyncTransfer({ version: 1, table: name, rows: values }) + frames.push(bytes) + directory.receipts.push({ + sequence: frames.length - 1, + table: name, + rows: values.length, + done: index === count - 1, + digest: hash(bytes) + }) + } + } + directory.pages = frames.length + directory.rows = rows.length + rehash(directory) + let request: SnapshotArchiveReaderRequest | undefined + const receipt = () => { + if (request === undefined) throw new Error('Fixture request has not been admitted') + return { + version: 1, + requestId: request.requestId, + expiresAt: request.notAfter, + state: 'ready', + archiveId: directory.archiveId, + digest: directory.digest + } + } + const response = (...[method, params]: Parameters): unknown => { + const input = params[0] as { + request?: SnapshotArchiveReaderRequest + sequence?: number + options?: { lifetimeMs: number; maxBytes: number } + } + switch (method) { + case 'getSnapshotArchiveReaderOffer': { + const serverTime = Date.now() + const fields = { + version: 2 as const, + nonce: 'a'.repeat(64), + notAfter: serverTime + input.options!.lifetimeMs, + maxBytes: input.options!.maxBytes + } + request = { ...fields, requestId: snapshotArchiveReaderRequestId(fields) } + directory.expiresAt = request.notAfter + return { + version: 1, + outcome: 'offered', + request, + offer: { + version: 1, + serverTime, + sourceStorageIdentityKey: expected.sourceStorageIdentityKey, + sourceSchema: binding.sourceSchema, + chain: 'test' + } + } + } + case 'admitSnapshotArchive': + if (input.request?.requestId !== request?.requestId) throw new Error('Fixture request was not offered') + return { version: 1, outcome: 'accepted', receipt: receipt() } + case 'getSnapshotArchiveStatus': + return receipt() + case 'getSnapshotArchiveDirectory': + return directory + case 'readSnapshotArchivePage': { + const sequence = input.sequence! + return { ...directory.receipts[sequence], bytes: frames[sequence] } + } + case 'cancelSnapshotArchiveRequest': + return true + default: + throw new Error(`Unexpected fixture operation ${method}`) + } + } + const implementation: SnapshotArchiveRpcCall = (method, params) => { + try { + return Promise.resolve(response(method, params)) + } catch (error) { + return Promise.reject(error) + } + } + const rpc = jest.fn(implementation) + const transport = new SnapshotArchiveTransport( + rpc, + expected.identityKey, + expected.sourceStorageIdentityKey, + 'test', + true + ) + return { transport, rpc, frames, directory, binding, receipt } +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveDirectoryFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveDirectoryFixtures.ts new file mode 100644 index 000000000..cc5f2bc4f --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveDirectoryFixtures.ts @@ -0,0 +1,85 @@ +import { createHash } from 'node:crypto' +import type { WalletSnapshotTable } from '../../src/storage/snapshot/WalletReadSnapshot' +import type { SnapshotArchiveDirectory } from '../../src/storage/snapshot/archive/SnapshotArchiveDirectory' + +export const now = Date.UTC(2026, 8, 30) +const identityKey = '02' + '11'.repeat(32) +export const sourceStorageIdentityKey = 'original-storage' +export const expected = { identityKey, chain: 'test' as const, sourceStorageIdentityKey } +export const tables: WalletSnapshotTable[] = [ + 'provenTxs', + 'provenTxReqs', + 'outputBaskets', + 'transactions', + 'commissions', + 'outputs', + 'outputTags', + 'outputTagMaps', + 'txLabels', + 'txLabelMaps', + 'certificates', + 'certificateFields', + 'syncStates' +] +export const hash = (value: string | Uint8Array) => createHash('sha256').update(value).digest('hex') + +export function fixture(extraPages = 0) { + const binding = { + version: 1, + snapshotId: 'a'.repeat(64), + sourceStorage: { + created_at: new Date(now - 1000).toISOString(), + updated_at: new Date(now).toISOString(), + storageIdentityKey: sourceStorageIdentityKey, + storageName: 'Original source', + chain: 'test', + dbtype: 'SQLite', + maxOutputScript: 100000 + }, + sourceSchema: '2026-09-30-002 add snapshot archive staging', + user: { + created_at: new Date(now - 1000).toISOString(), + updated_at: new Date(now).toISOString(), + userId: 7, + identityKey, + activeStorage: 'historical-primary' + } + } + const bindingJson = JSON.stringify(binding) + const directory: SnapshotArchiveDirectory = { + version: 1, + encoding: 'wallet-snapshot-rows/1', + archiveId: 'b'.repeat(64), + expiresAt: now + 1000, + pages: 13 + extraPages, + rows: extraPages, + digest: '', + bindingJson, + receipts: [] + } + const payloads: Uint8Array[] = [] + for (const table of tables) { + const count = table === 'provenTxs' ? extraPages + 1 : 1 + for (let page = 0; page < count; page++) { + const bytes = new TextEncoder().encode(`${table}:${page}`) + const done = page === count - 1 + payloads.push(bytes) + directory.receipts.push({ + sequence: directory.receipts.length, + table, + rows: done ? 0 : 1, + done, + digest: hash(bytes) + }) + } + } + rehash(directory) + return { directory, payloads, binding } +} + +export function rehash(directory: SnapshotArchiveDirectory): void { + let previous = hash(directory.bindingJson) + for (const page of directory.receipts) + previous = hash(JSON.stringify([previous, page.sequence, page.table, page.rows, page.done, page.digest])) + directory.digest = previous +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveFixtures.ts new file mode 100644 index 000000000..bebcfaaec --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveFixtures.ts @@ -0,0 +1,137 @@ +import type { StorageKnex } from '../../src/storage/StorageKnex' +import { runInSeries } from '../../src/utility/runInSeries' + +const identity = '02' + '11'.repeat(32) +const date = '2026-01-01T00:00:00.000Z' +const timestamp = { created_at: date, updated_at: date } + +export async function seedArchiveClosure(source: StorageKnex, userId: number, otherId: number): Promise { + const k = source.knex + await k('output_baskets').del() + await runInSeries([1, 2, 3], async id => { + const profileId = id === 2 ? otherId : userId + await k('proven_txs').insert({ + ...timestamp, + provenTxId: id, + txid: String(id).repeat(64), + height: id, + index: 0, + merklePath: Buffer.from([id, 0, 255]), + rawTx: Buffer.from([id, 1, 255]), + blockHash: 'a'.repeat(64), + merkleRoot: 'b'.repeat(64) + }) + await k('transactions').insert({ + ...timestamp, + transactionId: id, + userId: profileId, + provenTxId: id === 3 ? null : id, + status: 'completed', + reference: `tx-${id}`, + isOutgoing: true, + satoshis: 0, + description: `tx-${id}`, + txid: String(id).repeat(64), + rawTx: Buffer.from([id, 2, 255]), + inputBEEF: Buffer.from([id, 3, 255]) + }) + await k('proven_tx_reqs').insert({ + ...timestamp, + provenTxReqId: id, + provenTxId: id, + txid: String(id).repeat(64), + status: 'completed', + attempts: 0, + notified: true, + history: '{}', + notify: '{}', + rawTx: Buffer.from([id, 4, 255]), + wasBroadcast: true + }) + await k('output_baskets').insert({ + ...timestamp, + basketId: id, + userId: profileId, + name: `basket-${id}`, + isDeleted: id === 3 + }) + await k('outputs').insert({ + ...timestamp, + outputId: id, + userId: profileId, + transactionId: id, + basketId: id, + spendable: false, + change: true, + vout: 0, + satoshis: 1, + providedBy: 'you', + purpose: '', + type: 'P2PKH', + lockingScript: Buffer.from([id, 5, 255]) + }) + await k('commissions').insert({ + ...timestamp, + commissionId: id, + userId: profileId, + transactionId: id, + satoshis: 0, + keyOffset: 'offset', + isRedeemed: true, + lockingScript: Buffer.from([id, 6, 255]) + }) + await k('output_tags').insert({ + ...timestamp, + outputTagId: id, + userId: profileId, + tag: `tag-${id}`, + isDeleted: id === 3 + }) + await k('output_tags_map').insert({ ...timestamp, outputTagId: id, outputId: id, isDeleted: id === 3 }) + await k('tx_labels').insert({ + ...timestamp, + txLabelId: id, + userId: profileId, + label: `label-${id}`, + isDeleted: id === 3 + }) + await k('tx_labels_map').insert({ ...timestamp, txLabelId: id, transactionId: id, isDeleted: id === 3 }) + await k('certificates').insert({ + ...timestamp, + certificateId: id, + userId: profileId, + serialNumber: `serial-${id}`, + type: 'type', + certifier: identity, + subject: identity, + revocationOutpoint: 'a'.repeat(64) + '.0', + signature: 'signature', + isDeleted: id === 3 + }) + await runInSeries(['a', 'Z', 'é', '😀'], async fieldName => { + await k('certificate_fields').insert({ + ...timestamp, + certificateId: id, + userId: profileId, + fieldName, + fieldValue: `value-${id}`, + masterKey: 'key' + }) + }) + await k('sync_states').insert({ + ...timestamp, + syncStateId: id, + userId: profileId, + storageIdentityKey: `peer-${id}`, + storageName: `peer-${id}`, + status: 'unknown', + init: true, + refNum: `state-${id}`, + syncMap: '{}', + when: date + }) + }) + // Composite positions must handle repeated first keys and preserve deleted mappings. + await k('output_tags_map').insert({ ...timestamp, outputTagId: 1, outputId: 3, isDeleted: true }) + await k('tx_labels_map').insert({ ...timestamp, txLabelId: 1, transactionId: 3, isDeleted: true }) +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveHttpFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveHttpFixtures.ts new file mode 100644 index 000000000..499d24933 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotArchiveHttpFixtures.ts @@ -0,0 +1,133 @@ +import { mkdtemp, rm } from 'node:fs/promises' +import { once } from 'node:events' +import type { Server } from 'node:http' +import type { KnexSnapshotArchiveRpc } from '../../src/storage/snapshot/archive/KnexSnapshotArchiveRpc' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { knex } from 'knex' +import { PrivateKey, ProtoWallet } from '@bsv/sdk' +import type { Wallet } from '../../src/Wallet' +import { StorageKnex } from '../../src/storage/StorageKnex' +import { StorageProvider } from '../../src/storage/StorageProvider' +import { + StorageServer, + type WalletStorageServerOptions +} from '../../src/storage/remoting/StorageServer' +import { seedArchiveClosure } from './snapshotArchiveFixtures' +import { runInSeries } from '../../src/utility/runInSeries' + +export function gate() { + let resolve!: () => void + const promise = new Promise(done => { + resolve = done + }) + return { promise, resolve } +} + +async function closeFixtureArchiveRpc(server: StorageServer): Promise { + const rpc = Reflect.get(server, 'snapshotArchives') as KnexSnapshotArchiveRpc | undefined + await rpc?.close() +} +async function closeFixtureListener(server: StorageServer): Promise { + const listener = server.server as Server | undefined + if (!listener?.listening) return + await new Promise((resolve, reject) => { + listener.close((error?: NodeJS.ErrnoException) => { + if (error !== undefined && error.code !== 'ERR_SERVER_NOT_RUNNING') reject(error) + else resolve() + }) + }) +} + +export async function snapshotHttpFixture(snapshotSync = true, seedClosure = true) { + const directory = await mkdtemp(join(tmpdir(), 'snapshot-http-')) + const key = PrivateKey.fromRandom() + const identityKey = key.toPublicKey().toString() + const wallet = new ProtoWallet(key) + const serverKey = PrivateKey.fromRandom() + const serverWallet = Object.assign(new ProtoWallet(serverKey), { + chain: 'test' + }) as unknown as Wallet + const storage = new StorageKnex({ + ...StorageProvider.createStorageBaseOptions('test'), + snapshotSync, + knex: knex({ + client: 'better-sqlite3', + connection: { filename: join(directory, 'wallet.sqlite') }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + }) + const servers: StorageServer[] = [] + const close = async () => { + // Test the public close outcome, but own the actual resources independently. + // A failed or mutated close must not leak a listener/capture into later tests. + const draining = servers.flatMap(server => [ + (async () => await server.close())(), + closeFixtureArchiveRpc(server), + closeFixtureListener(server) + ]) + const results = await Promise.allSettled(draining) + const failures = results + .filter(result => result.status === 'rejected') + .map(result => result.reason) + await runInSeries( + [ + async () => await storage.destroy(), + async () => await storage.knex.destroy(), + async () => await rm(directory, { recursive: true, force: true }) + ], + async cleanup => { + try { + await cleanup() + } catch (error) { + failures.push(error) + } + } + ) + if (failures.length === 1) throw failures[0] + if (failures.length) throw new AggregateError(failures, 'Snapshot HTTP fixture cleanup failed') + } + try { + await storage.knex.raw('PRAGMA journal_mode = WAL') + await storage.migrate('HTTP snapshot source', 'http-snapshot-source') + await storage.makeAvailable() + const { user } = await storage.findOrInsertUser(identityKey) + const { user: other } = await storage.findOrInsertUser( + PrivateKey.fromRandom().toPublicKey().toString() + ) + if (seedClosure) await seedArchiveClosure(storage, user.userId, other.userId) + const serve = async (options: Partial = {}) => { + const server = new StorageServer(storage, { + port: 0, + host: '127.0.0.1', + wallet: serverWallet, + monetize: false, + logRpcRequests: false, + ...options + }) + servers.push(server) + server.start() + if (!server.server.listening) await once(server.server, 'listening') + const address = server.server.address() + if (address === null || typeof address === 'string') + throw new Error('Fixture listener did not bind') + return { server, url: `http://127.0.0.1:${address.port}` } + } + return { + storage, + identityKey, + wallet, + serverIdentityKey: serverKey.toPublicKey().toString(), + serve, + close + } + } catch (error) { + try { + await close() + } catch (error_) { + throw new AggregateError([error, error_], 'Snapshot HTTP fixture setup and cleanup failed') + } + throw error + } +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotCertificateFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotCertificateFixtures.ts new file mode 100644 index 000000000..fa03e9222 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotCertificateFixtures.ts @@ -0,0 +1,139 @@ +import assert from 'node:assert/strict' +import { knex, type Knex } from 'knex' +import { runInSeries } from '../../src/utility/runInSeries' + +// Independently defined source graph and SQL union oracle; no migration registry +// or auxiliary trigger expression is used to derive expected membership. +export const certificateFieldNames = ['', 'A', 'a', 'é', 'e\u0301', 'Ω', '名字', '🚲', 'a ', 'A\0B', '😀'.repeat(100)] +export interface CertificateOperation { + kind: number + id: number + user: number + name: number +} + +export async function createCertificateSource(k: Knex, collation = 'BINARY'): Promise { + const mysql = String(k.client.config.client).includes('mysql') + assert.ok(['BINARY', 'NOCASE', 'RTRIM'].includes(collation)) + await k.raw( + mysql + ? 'CREATE TABLE certificates(certificateId INT UNSIGNED PRIMARY KEY,userId INT UNSIGNED NOT NULL,isDeleted TINYINT NOT NULL DEFAULT 0) ENGINE=InnoDB' + : 'CREATE TABLE certificates(certificateId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,isDeleted INTEGER NOT NULL DEFAULT 0)' + ) + await k.raw( + mysql + ? 'CREATE TABLE certificate_fields(userId INT UNSIGNED NOT NULL,fieldName VARCHAR(100) NOT NULL,certificateId INT UNSIGNED NOT NULL,fieldValue VARCHAR(255) NOT NULL,UNIQUE(fieldName,certificateId)) ENGINE=InnoDB' + : `CREATE TABLE certificate_fields(userId INTEGER NOT NULL,fieldName VARCHAR(100) COLLATE ${collation} NOT NULL,certificateId INTEGER NOT NULL,fieldValue VARCHAR(255) NOT NULL,UNIQUE(fieldName,certificateId))` + ) +} + +export async function minimalCertificateDatabase(collation = 'BINARY'): Promise { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + try { + await createCertificateSource(k, collation) + return k + } catch (error) { + await k.destroy() + throw error + } +} + +export async function expectCertificateMembership(k: Knex): Promise { + const result = await k.raw( + 'SELECT userId,fieldName,certificateId,SUM(bit) AS membership FROM (SELECT userId,fieldName,certificateId,1 AS bit FROM certificate_fields UNION ALL SELECT c.userId,f.fieldName,f.certificateId,2 AS bit FROM certificate_fields f JOIN certificates c ON c.certificateId=f.certificateId) owned_fields GROUP BY userId,fieldName,certificateId ORDER BY userId,fieldName,certificateId' + ) + const rows: Array<{ userId: number; fieldName: string; certificateId: number; membership: number | string }> = String( + k.client.config.client + ).includes('mysql') + ? result[0] + : result + const expected = rows.map(row => ({ ...row, membership: Number(row.membership) })) + const actual = await k('snapshot_certificate_field_keys') + .select({ + userId: 'snapshotUserId', + fieldName: 'snapshotFieldName', + certificateId: 'snapshotCertificateId', + membership: 'snapshotMembership' + }) + .orderBy(['snapshotUserId', 'snapshotFieldName', 'snapshotCertificateId']) + expect(actual).toEqual(expected) +} + +export async function seedCertificateFields(k: Knex): Promise { + await k('certificate_fields').delete() + await k('certificates').delete() + await k('certificates').insert([ + { certificateId: 1, userId: 1 }, + { certificateId: 2, userId: 2 } + ]) + await runInSeries(certificateFieldNames.entries(), async ([i, fieldName]) => { + await k('certificate_fields') + .insert({ userId: (i % 2) + 1, fieldName, certificateId: (i % 2) + 1, fieldValue: 'initial' }) + .onConflict(['fieldName', 'certificateId']) + .merge(['userId', 'fieldValue']) + }) +} + +export async function applyCertificateOperation(k: Knex, op: CertificateOperation): Promise { + const fieldName = certificateFieldNames[op.name], + certificateId = op.id + const selected = () => k('certificate_fields').where({ fieldName, certificateId }) + switch (op.kind) { + case 0: + await k('certificates').insert({ certificateId, userId: op.user }).onConflict('certificateId').merge(['userId']) + break + case 1: + await k('certificates').where({ certificateId }).delete() + break + case 2: + await k('certificate_fields') + .insert({ userId: op.user, fieldName, certificateId, fieldValue: 'value-' + op.user }) + .onConflict(['fieldName', 'certificateId']) + .merge(['userId', 'fieldValue']) + break + case 3: + await selected().delete() + break + case 4: + await selected().update({ userId: op.user }) + break + case 5: + if ( + !(await k('certificates') + .where({ certificateId: certificateId + 4 }) + .first()) + ) + await k('certificates') + .where({ certificateId }) + .update({ certificateId: certificateId + 4 }) + break + case 6: + try { + await selected().update({ fieldName: certificateFieldNames[(op.name + 1) % certificateFieldNames.length] }) + } catch (error) { + if (!['ER_DUP_ENTRY', 'SQLITE_CONSTRAINT_UNIQUE'].includes((error as { code: string }).code)) throw error + } + break + case 7: + await assert.rejects( + k.transaction(async trx => { + await trx('certificate_fields').where({ fieldName, certificateId }).delete() + throw new Error('synthetic operation rollback') + }), + /synthetic operation rollback/ + ) + break + case 8: + await k('certificates') + .where({ certificateId }) + .update({ isDeleted: k.raw('1-isDeleted') }) + break + default: + throw new Error('Unknown certificate fixture operation') + } +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotGlobalFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotGlobalFixtures.ts new file mode 100644 index 000000000..97200cda0 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotGlobalFixtures.ts @@ -0,0 +1,145 @@ +import { knex, type Knex } from 'knex' +import { runInSeries } from '../../src/utility/runInSeries' + +/** Independent standard-source graph; auxiliary definitions are not its oracle. */ +export async function createGlobalSource(k: Knex, collation = 'BINARY'): Promise { + if (!['BINARY', 'NOCASE', 'RTRIM'].includes(collation)) throw new Error('Unsupported fixture collation') + const mysql = String(k.client.config.client).includes('mysql') + const integer = mysql ? 'INT UNSIGNED' : 'INTEGER' + const text = mysql ? 'VARCHAR(64)' : `VARCHAR(64) COLLATE ${collation}` + const suffix = mysql ? ' ENGINE=InnoDB' : '' + await k.raw(`CREATE TABLE proven_txs(provenTxId ${integer} NOT NULL PRIMARY KEY)${suffix}`) + await k.raw( + `CREATE TABLE proven_tx_reqs(provenTxReqId ${integer} NOT NULL PRIMARY KEY,provenTxId ${integer},txid ${text} NOT NULL UNIQUE)${suffix}` + ) + await k.raw( + `CREATE TABLE transactions(transactionId ${integer} NOT NULL PRIMARY KEY,userId ${integer} NOT NULL,provenTxId ${integer},txid ${text})${suffix}` + ) + await k.schema.alterTable('transactions', table => { + void table.index('txid') + }) +} + +export async function minimalGlobalDatabase(collation = 'BINARY'): Promise { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + try { + await createGlobalSource(k, collation) + return k + } catch (error) { + await k.destroy() + throw error + } +} + +interface Edge { + transactionId: number + requestId: number + tableId: number + rowId: number + userId: number +} +export async function expectGlobalMembership(k: Knex): Promise { + const direct: Edge[] = await k('transactions') + .select(k.raw('transactionId,0 AS requestId,1 AS tableId,provenTxId AS rowId,userId')) + .whereNotNull('provenTxId') + const requests: Edge[] = await k('transactions AS t') + .join('proven_tx_reqs AS r', 'r.txid', 't.txid') + .select(k.raw('t.transactionId,r.provenTxReqId AS requestId,0 AS tableId,r.provenTxReqId AS rowId,t.userId')) + const indirect: Edge[] = await k('transactions AS t') + .join('proven_tx_reqs AS r', 'r.txid', 't.txid') + .select(k.raw('t.transactionId,r.provenTxReqId AS requestId,1 AS tableId,r.provenTxId AS rowId,t.userId')) + .whereNotNull('r.provenTxId') + const expected = [...direct, ...requests, ...indirect] + const order = (rows: Edge[]) => + rows + .map(row => [row.transactionId, row.requestId, row.tableId, row.rowId, row.userId]) + .sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))) + expect(order(await k('snapshot_global_edges'))).toEqual(order(expected)) + const proofs = new Set((await k('proven_txs').select('provenTxId')).map(row => row.provenTxId)) + const counts = new Map() + for (const row of expected) { + const key = JSON.stringify([row.tableId, row.userId, row.rowId]) + counts.set(key, (counts.get(key) ?? 0) + 1) + } + const actual = await k('snapshot_global_keys') + expect(actual).toHaveLength(counts.size) + for (const row of actual) { + expect(Number(row.refs)).toBe(counts.get(JSON.stringify([row.tableId, row.userId, row.rowId]))) + expect(Number(row.present)).toBe(row.tableId === 0 || proofs.has(row.rowId) ? 1 : 0) + } + const guards = await k('snapshot_global_guards').orderBy('proofId') + expect(guards.map(row => row.proofId)).toEqual( + [...new Set(expected.filter(row => row.tableId === 1).map(row => row.rowId))].sort((a, b) => a - b) + ) + for (const row of guards) expect(Number(row.present)).toBe(proofs.has(row.proofId) ? 1 : 0) +} + +export interface GlobalOperation { + kind: number + id: number + userId: number + otherId: number + nullable: boolean +} +export async function clearGlobalSource(k: Knex): Promise { + await runInSeries(['transactions', 'proven_tx_reqs', 'proven_txs'], async table => { + await k(table).delete() + }) +} +export async function applyGlobalOperation(k: Knex, op: GlobalOperation): Promise { + const provenTxId = op.nullable ? null : op.otherId + const txid = op.nullable ? null : 'r' + op.otherId + try { + switch (op.kind) { + case 0: + await k('transactions') + .insert({ transactionId: op.id, userId: op.userId, txid, provenTxId }) + .onConflict('transactionId') + .merge() + break + case 1: + await k('transactions').where('transactionId', op.id).delete() + break + case 2: + await k('transactions').where('transactionId', op.id).update({ userId: op.userId, txid, provenTxId }) + break + case 3: + await k('proven_tx_reqs') + .insert({ provenTxReqId: op.id, txid: 'r' + op.id, provenTxId }) + .onConflict('provenTxReqId') + .merge() + break + case 4: + await k('proven_tx_reqs').where('provenTxReqId', op.id).delete() + break + case 5: + await k('proven_tx_reqs') + .where('provenTxReqId', op.id) + .update({ txid: 'r' + op.otherId, provenTxId }) + break + case 6: + await k('proven_txs').insert({ provenTxId: op.id }).onConflict('provenTxId').ignore() + break + case 7: + await k('proven_txs').where('provenTxId', op.id).delete() + break + case 8: + await k('proven_txs').where('provenTxId', op.id).update({ provenTxId: op.otherId }) + break + default: + throw new Error('Unsupported generated operation') + } + } catch (error) { + if ( + !['ER_DUP_ENTRY', 'SQLITE_CONSTRAINT_PRIMARYKEY', 'SQLITE_CONSTRAINT_UNIQUE'].includes( + (error as { code?: string }).code ?? '' + ) + ) + throw error + } +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotHistoricalMigrations.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotHistoricalMigrations.ts new file mode 100644 index 000000000..a6faee332 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotHistoricalMigrations.ts @@ -0,0 +1,21 @@ +import { StorageKnex } from '../../src/storage/StorageKnex' +import { KnexMigrations, SNAPSHOT_SQLITE_INDEX_MIGRATION } from '../../src/storage/schema/KnexMigrations' + +/** Deliberately build the historical schema whose migration a fixture exercises. */ +export async function migrateBeforeSqliteGeneration( + source: StorageKnex, + storageName: string, + storageIdentityKey: string +): Promise { + const migrationSource = new KnexMigrations(source.chain, storageName, storageIdentityKey, 1024) + delete migrationSource.migrations[SNAPSHOT_SQLITE_INDEX_MIGRATION] + const config = { migrationSource, disableTransactions: false } + const sqlite = String(source.knex.client.config.client).includes('sqlite') + if (sqlite) await source.knex.raw('PRAGMA foreign_keys=OFF') + try { + await source.knex.migrate.latest(config) + return await source.knex.migrate.currentVersion(config) + } finally { + if (sqlite) await source.knex.raw('PRAGMA foreign_keys=ON') + } +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotRelationFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotRelationFixtures.ts new file mode 100644 index 000000000..93f3b25c3 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotRelationFixtures.ts @@ -0,0 +1,85 @@ +import { knex, type Knex } from 'knex' +import { runInSeries } from '../../src/utility/runInSeries' + +// Independent fixture definitions and source-derived oracle, rather than the +// migration's relation registry or trigger expressions. +export const relationFixtures = [ + { table: 'tx_labels_map', left: 'tx_labels', leftKey: 'txLabelId', right: 'transactions', rightKey: 'transactionId' }, + { table: 'output_tags_map', left: 'output_tags', leftKey: 'outputTagId', right: 'outputs', rightKey: 'outputId' } +] as const + +export async function minimalRelationDatabase(): Promise { + const k = knex({ + client: 'better-sqlite3', + connection: { filename: ':memory:' }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + try { + await runInSeries(relationFixtures, async relation => { + await runInSeries( + [ + [relation.left, relation.leftKey], + [relation.right, relation.rightKey] + ], + async ([table, key]) => { + await k.schema.createTable(table, t => { + t.integer(key).primary() + t.integer('userId').notNullable() + }) + } + ) + await k.schema.createTable(relation.table, t => { + t.integer(relation.leftKey) + t.integer(relation.rightKey) + t.boolean('isDeleted') + t.primary([relation.leftKey, relation.rightKey]) + t.index(relation.rightKey) + }) + }) + return k + } catch (error) { + await k.destroy() + throw error + } +} + +export async function expectRelationMembership(k: Knex): Promise { + const expected: Array> = [] + await runInSeries(relationFixtures.entries(), async ([tableId, relation]) => { + const left = new Map((await k(relation.left)).map(row => [row[relation.leftKey], row.userId])) + const right = new Map((await k(relation.right)).map(row => [row[relation.rightKey], row.userId])) + for (const row of await k(relation.table)) { + const owners = new Map() + for (const [userId, bit] of [ + [left.get(row[relation.leftKey]), 1], + [right.get(row[relation.rightKey]), 2] + ]) { + if (userId !== undefined && bit !== undefined) owners.set(userId, (owners.get(userId) ?? 0) | bit) + } + for (const [userId, membership] of owners) + expected.push({ + snapshotTableId: tableId, + snapshotUserId: userId, + snapshotLeftId: row[relation.leftKey], + snapshotRightId: row[relation.rightKey], + snapshotMembership: membership + }) + } + }) + expected.sort( + (a, b) => + a.snapshotTableId - b.snapshotTableId || + a.snapshotUserId - b.snapshotUserId || + a.snapshotLeftId - b.snapshotLeftId || + a.snapshotRightId - b.snapshotRightId + ) + expect( + await k('snapshot_relation_keys').orderBy([ + 'snapshotTableId', + 'snapshotUserId', + 'snapshotLeftId', + 'snapshotRightId' + ]) + ).toEqual(expected) +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteFixtures.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteFixtures.ts new file mode 100644 index 000000000..3a0470aff --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteFixtures.ts @@ -0,0 +1,160 @@ +import { runInSeries } from '../../src/utility/runInSeries' +import { knex, type Knex } from 'knex' +import { installMembershipDraft } from './snapshotSqliteMaintenanceFixture' +import { addSnapshotProfileIndexes } from '../../src/storage/schema/snapshotProfileIndexMigration' +import { addSnapshotRelationIndexes } from '../../src/storage/schema/snapshotRelationIndexMigration' +import { addSnapshotCertificateIndexes } from '../../src/storage/schema/snapshotCertificateIndexMigration' +import { addSnapshotGlobalIndexes } from '../../src/storage/schema/snapshotGlobalIndexMigration' +import { expectRelationMembership } from './snapshotRelationFixtures' +import { expectCertificateMembership } from './snapshotCertificateFixtures' +import { expectGlobalMembership } from './snapshotGlobalFixtures' + +export const profiles = [ + ['transactions', 'transactionId'], + ['outputs', 'outputId'], + ['certificates', 'certificateId'], + ['tx_labels', 'txLabelId'], + ['output_baskets', 'basketId'], + ['output_tags', 'outputTagId'], + ['commissions', 'commissionId'], + ['sync_states', 'syncStateId'] +] +export const numeric = [...profiles, ['proven_txs', 'provenTxId'], ['proven_tx_reqs', 'provenTxReqId']] +export const tables = [...numeric.map(([table]) => table), 'tx_labels_map', 'output_tags_map', 'certificate_fields'] + +export async function fixture(collation: string, recursive: boolean, corrected = true, filename = ':memory:') { + const k = knex({ + client: 'better-sqlite3', + connection: { filename }, + useNullAsDefault: true, + pool: { min: 1, max: 1 } + }) + const schema = [ + 'proven_txs(provenTxId INTEGER PRIMARY KEY,txid VARCHAR(64) NOT NULL UNIQUE)', + 'proven_tx_reqs(provenTxReqId INTEGER PRIMARY KEY,txid VARCHAR(64) NOT NULL UNIQUE,provenTxId INTEGER)', + 'transactions(transactionId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,reference VARCHAR(64) NOT NULL UNIQUE,txid VARCHAR(64),provenTxId INTEGER)', + 'outputs(outputId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,transactionId INTEGER NOT NULL,vout INTEGER NOT NULL,UNIQUE(transactionId,vout,userId))', + 'certificates(certificateId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,type VARCHAR(100) NOT NULL,certifier VARCHAR(130) NOT NULL,serialNumber VARCHAR(100) NOT NULL,UNIQUE(userId,type,certifier,serialNumber))', + 'tx_labels(txLabelId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,label VARCHAR(300) NOT NULL,UNIQUE(label,userId))', + 'output_tags(outputTagId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,tag VARCHAR(150) NOT NULL,UNIQUE(tag,userId))', + 'output_baskets(basketId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,name VARCHAR(300) NOT NULL,UNIQUE(name,userId))', + 'commissions(commissionId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,transactionId INTEGER NOT NULL UNIQUE)', + 'sync_states(syncStateId INTEGER PRIMARY KEY,userId INTEGER NOT NULL,refNum VARCHAR(100) NOT NULL UNIQUE)', + 'tx_labels_map(txLabelId INTEGER NOT NULL,transactionId INTEGER NOT NULL,UNIQUE(txLabelId,transactionId))', + 'output_tags_map(outputTagId INTEGER NOT NULL,outputId INTEGER NOT NULL,UNIQUE(outputTagId,outputId))', + 'certificate_fields(userId INTEGER NOT NULL,fieldName VARCHAR(100) NOT NULL,certificateId INTEGER NOT NULL,fieldValue VARCHAR(255) NOT NULL,UNIQUE(fieldName,certificateId))' + ] + try { + await k.raw('PRAGMA recursive_triggers=' + Number(recursive)) + if (filename !== ':memory:') await k.raw('PRAGMA journal_mode=WAL') + await runInSeries(schema, async sql => { + await k.raw('CREATE TABLE ' + sql.replaceAll(/VARCHAR\(\d+\)/g, type => type + ' COLLATE ' + collation)) + }) + await k.schema.alterTable('transactions', table => { + void table.index('txid') + }) + await k.schema.alterTable('tx_labels_map', table => { + void table.index('transactionId') + }) + await k.schema.alterTable('output_tags_map', table => { + void table.index('outputId') + }) + await addSnapshotProfileIndexes(k) + await addSnapshotRelationIndexes(k) + await addSnapshotCertificateIndexes(k) + await addSnapshotGlobalIndexes(k) + await k.schema.createTable('knex_migrations', table => { + table.increments('id') + table.string('name').unique().notNullable() + table.integer('batch').notNullable() + table.timestamp('migration_time').notNullable() + }) + await k('knex_migrations').insert( + [ + '2026-10-01-003 add snapshot profile key indexes', + '2026-10-01-004 add snapshot relation key indexes', + '2026-10-01-005 add snapshot certificate field key indexes', + '2026-10-01-006 add snapshot global reference indexes' + ].map(name => ({ name, batch: 1, migration_time: new Date('2026-01-01') })) + ) + if (corrected) await installMembershipDraft(k) + return k + } catch (error) { + await k.destroy() + throw error + } +} + +export async function exact(k: Knex) { + const expected: Array<{ + snapshotTableId: number + snapshotUserId: number + snapshotRowId: number + }> = [] + await runInSeries(profiles.entries(), async ([id, [table, key]]) => { + const rows = await k(table) + for (const row of rows) expected.push({ snapshotTableId: id, snapshotUserId: row.userId, snapshotRowId: row[key] }) + }) + expected.sort( + (a, b) => + a.snapshotTableId - b.snapshotTableId || a.snapshotUserId - b.snapshotUserId || a.snapshotRowId - b.snapshotRowId + ) + expect(await k('snapshot_profile_keys').orderBy(['snapshotTableId', 'snapshotUserId', 'snapshotRowId'])).toEqual( + expected + ) + await expectRelationMembership(k) + await expectCertificateMembership(k) + await expectGlobalMembership(k) +} + +export function value(table: string, id: number, other: number, userId: number): Record { + const key = numeric.find(([name]) => name === table)?.[1] + const row = key ? { [key]: id, ...(profiles.some(([name]) => name === table) ? { userId } : {}) } : {} + switch (table) { + case 'transactions': + return { ...row, reference: 'r' + other, txid: 't' + other, provenTxId: other } + case 'outputs': + return { ...row, transactionId: other, vout: id % 2 } + case 'certificates': + return { ...row, type: 'a', certifier: 'b', serialNumber: 's' + other } + case 'tx_labels': + return { ...row, label: 'l' + other } + case 'output_tags': + return { ...row, tag: 'g' + other } + case 'output_baskets': + return { ...row, name: 'b' + other } + case 'commissions': + return { ...row, transactionId: other } + case 'sync_states': + return { ...row, refNum: 'r' + other } + case 'proven_txs': + return { ...row, txid: 't' + other } + case 'proven_tx_reqs': + return { ...row, txid: 't' + other, provenTxId: other } + case 'tx_labels_map': + return { txLabelId: id, transactionId: other } + case 'output_tags_map': + return { outputTagId: id, outputId: other } + case 'certificate_fields': + return { + userId, + fieldName: ['a', 'A', 'a ', 'é', 'A\0B', '名字'][id - 1], + certificateId: other, + fieldValue: 'p' + } + default: + throw new Error('Unknown fixture table') + } +} +export function keyOf(table: string, id: number, other: number) { + const key = numeric.find(([name]) => name === table)?.[1] + if (key) return { [key]: id } + const result = value(table, id, other, 1) + delete result.userId + delete result.fieldValue + return result +} +export async function replace(k: Knex, table: string, row: Record) { + const query = k(table).insert(row).toSQL() + await k.raw(query.sql.replace(/^insert/i, 'INSERT OR REPLACE'), query.bindings) +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteIdentityFixture.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteIdentityFixture.ts new file mode 100644 index 000000000..03fcadc3e --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteIdentityFixture.ts @@ -0,0 +1,27 @@ +import { runInSeries } from '../../src/utility/runInSeries' +import type { Knex } from 'knex' +import type { IdentityDefinition } from '../../src/storage/schema/snapshotSqliteIdentity' +import { WERR_INVALID_OPERATION } from '../../src/sdk/WERR_errors' + +/** Bound the background copy by the numeric source key; writes share its transaction. */ +export async function copyIdentityPage( + k: Knex, + identity: IdentityDefinition, + after: number, + count = 256 +): Promise { + if (!k.isTransaction) throw new WERR_INVALID_OPERATION('Identity bootstrap requires a transaction') + if (!Number.isSafeInteger(after) || after < 0 || !Number.isSafeInteger(count) || count < 1 || count > 256) + throw new WERR_INVALID_OPERATION('Invalid identity bootstrap position') + const columns = identity.columns.map(column => column.name) + const rows = await k(identity.source.table) + .select(columns) + .where(identity.source.key, '>', after) + .orderBy(identity.source.key) + .limit(count) + await runInSeries(rows, async row => { + const primary = [identity.source.key, ...(identity.source.owner ? [identity.source.owner] : [])] + await k(identity.table).insert(row).onConflict(primary).merge() + }) + return rows.length === count ? Number(rows.at(-1)[identity.source.key]) : undefined +} diff --git a/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteMaintenanceFixture.ts b/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteMaintenanceFixture.ts new file mode 100644 index 000000000..483e17c01 --- /dev/null +++ b/packages/wallet/wallet-toolbox/test/utils/snapshotSqliteMaintenanceFixture.ts @@ -0,0 +1,34 @@ +import { runInSeries } from '../../src/utility/runInSeries' +import type { Knex } from 'knex' +import { readIdentity, identityDDL, type IdentityDefinition } from '../../src/storage/schema/snapshotSqliteIdentity' +import { numeric, relations, membershipTriggers } from '../../src/storage/schema/snapshotSqliteMembership' + +export async function installMembershipDraft(k: Knex): Promise { + const definitions: IdentityDefinition[] = [] + await runInSeries(numeric, async source => { + definitions.push(await readIdentity(k, source)) + }) + await k.transaction(async trx => { + await runInSeries(definitions, async definition => { + await runInSeries(identityDDL(definition), async ddl => { + await trx.raw(ddl) + }) + }) + const sources = [ + ...numeric.map(source => source.table), + ...relations.map(relation => relation.table), + 'certificate_fields' + ] + const old: Array<{ name: string }> = await trx('sqlite_master') + .where('type', 'trigger') + .whereIn('tbl_name', sources) + .select('name') + await runInSeries(old, async trigger => { + if (/^snapshot_(profile|relation|certificate|global)_/.test(trigger.name)) + await trx.raw('DROP TRIGGER ??', [trigger.name]) + }) + await runInSeries(membershipTriggers(definitions), async sql => { + await trx.raw(sql) + }) + }) +} diff --git a/patches/metro-file-map@0.87.1.patch b/patches/metro-file-map@0.87.1.patch new file mode 100644 index 000000000..3223d791c --- /dev/null +++ b/patches/metro-file-map@0.87.1.patch @@ -0,0 +1,84 @@ +diff --git a/src/watchers/common.js b/src/watchers/common.js +--- a/src/watchers/common.js ++++ b/src/watchers/common.js +@@ -11,7 +11,7 @@ + exports.includedByGlob = includedByGlob; + exports.posixPathMatchesPattern = void 0; + exports.typeFromStat = typeFromStat; +-var _micromatch = _interopRequireDefault(require("micromatch")); ++var _picomatch = _interopRequireDefault(require("picomatch")); + var _nodePath = _interopRequireDefault(require("node:path")); + function _interopRequireDefault(e) { + return e && e.__esModule ? e : { default: e }; +@@ -20,11 +20,21 @@ + const TOUCH_EVENT = (exports.TOUCH_EVENT = "touch"); + const RECRAWL_EVENT = (exports.RECRAWL_EVENT = "recrawl"); + const ALL_EVENT = (exports.ALL_EVENT = "all"); ++// Keep Metro's matching semantics using micromatch.some's existing matcher. ++// The unused brace expansion package is excluded from this build-tool closure. ++function matchesAny(relativePath, patterns, options) { ++ for (const pattern of typeof patterns === "string" ? [patterns] : patterns) { ++ if (_picomatch.default(String(pattern), options)(relativePath)) { ++ return true; ++ } ++ } ++ return false; ++} + function includedByGlob(type, globs, dot, relativePath) { + if (globs.length === 0 || type !== "f") { +- return dot || _micromatch.default.some(relativePath, "**/*"); ++ return dot || matchesAny(relativePath, "**/*"); + } +- return _micromatch.default.some(relativePath, globs, { ++ return matchesAny(relativePath, globs, { + dot, + }); + } +diff --git a/src/watchers/common.js.flow b/src/watchers/common.js.flow +--- a/src/watchers/common.js.flow ++++ b/src/watchers/common.js.flow +@@ -17,8 +17,8 @@ + import type {ChangeEventMetadata} from '../flow-types'; + import type {Stats} from 'node:fs'; + +-// $FlowFixMe[untyped-import] - Write libdefs for `micromatch` +-import micromatch from 'micromatch'; ++// $FlowFixMe[untyped-import] - Write libdefs for `picomatch` ++import picomatch from 'picomatch'; + import path from 'node:path'; + + /** +@@ -38,6 +38,21 @@ + watchmanPath?: string, + }>; + ++// Keep Metro's matching semantics using micromatch.some's existing matcher. ++// The unused brace expansion package is excluded from this build-tool closure. ++function matchesAny( ++ relativePath: string, ++ patterns: string | ReadonlyArray, ++ options?: Readonly<{dot?: boolean}>, ++): boolean { ++ for (const pattern of typeof patterns === 'string' ? [patterns] : patterns) { ++ if (picomatch(String(pattern), options)(relativePath)) { ++ return true; ++ } ++ } ++ return false; ++} ++ + /** + * Checks a file relative path against the globs array. + */ +@@ -50,9 +65,9 @@ + // For non-regular files or if there are no glob matchers, just respect the + // `dot` option to filter dotfiles if dot === false. + if (globs.length === 0 || type !== 'f') { +- return dot || micromatch.some(relativePath, '**/*'); ++ return dot || matchesAny(relativePath, '**/*'); + } +- return micromatch.some(relativePath, globs, {dot}); ++ return matchesAny(relativePath, globs, {dot}); + } + + /** diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 465d24f83..01eebae25 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -6,13 +6,21 @@ settings: overrides: '@chainsafe/libp2p-yamux>@libp2p/utils': 7.4.1 - brace-expansion@<5.0.11: 5.0.11 + brace-expansion@<5.0.12: 5.0.12 engine.io@<6.6.10: 6.6.10 nanoid@<3.3.18: 3.3.18 toml@<4.2.0: 4.2.0 qs@<6.16.0: 6.16.0 js-yaml@<3.15.2: 3.15.2 lodash-es@<4.18.0: 4.18.1 + metro-file-map@0.87.1>micromatch: '-' + +packageExtensionsChecksum: sha256-j1ipTIriuh1r8NrHzy5rLoGNr081FjMxVoEY/IdY3mE= + +patchedDependencies: + metro-file-map@0.87.1: + hash: 28741c2833798bbfda1432bd62f13f338d31f12a6161f4268a6cc6f01022018b + path: patches/metro-file-map@0.87.1.patch importers: @@ -927,9 +935,15 @@ importers: '@typescript/native': specifier: npm:typescript@7.0.2 version: typescript@7.0.2 + esbuild: + specifier: 0.28.2 + version: 0.28.2 express: specifier: ^5.2.1 version: 5.2.1 + express-rate-limit: + specifier: 8.6.1 + version: 8.6.1(express@5.2.1) fast-check: specifier: ^4.9.0 version: 4.10.1 @@ -939,6 +953,9 @@ importers: oxlint: specifier: ^1.83.0 version: 1.83.0 + puppeteer-core: + specifier: ^25.4.0 + version: 25.11.0(yauzl@3.4.0) ts-jest: specifier: ^29.4.12 version: 29.4.12(@babel/core@7.29.7)(@jest/transform@30.5.1)(@jest/types@30.5.1)(@typescript/typescript6@6.0.2)(babel-jest@30.5.1(@babel/core@7.29.7))(esbuild@0.28.2)(jest-util@30.5.1)(jest@30.5.1(@types/node@26.6.1)(ts-node@10.9.2(@types/node@26.6.1)(@typescript/typescript6@6.0.2))) @@ -1897,6 +1914,9 @@ importers: oxlint: specifier: ^1.83.0 version: 1.83.0 + puppeteer-core: + specifier: ^25.4.0 + version: 25.11.0(yauzl@3.4.0) tsdown: specifier: 0.23.0 version: 0.23.0(@arethetypeswrong/core@0.18.5)(@typescript/typescript6@6.0.2)(publint@0.3.24)(tsx@4.23.13) @@ -5069,14 +5089,10 @@ packages: resolution: {integrity: sha512-2cGmJupaNgg+QUwVLAucDuWuoMZ6EX9iHDRswZ5lsNYEmwPaRknMPCLZz07yTzVq/83p4o/wzbDZbBrTvGGTIw==} engines: {node: '>=18'} - brace-expansion@5.0.11: - resolution: {integrity: sha512-awigjhi6cLTh90bdw6+QJ9CtmJmyYhEIi70iCbc8Rozn04Fw9FeQIBjv/E22FFGuCGx1bLJyUfB64x/szUSXUg==} + brace-expansion@5.0.12: + resolution: {integrity: sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ==} engines: {node: 20 || >=22} - braces@3.0.3: - resolution: {integrity: sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA==} - engines: {node: '>=8'} - browserslist@4.29.0: resolution: {integrity: sha512-3GSvyjvDI4Dur1Meg2BekJquu5uF+9R9a1+5M1Mde192eZoXbeXjzgOsgqPS2V8D5wrrip0gR5Hf/GhWQ9ZzaA==} engines: {node: ^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7} @@ -5640,8 +5656,8 @@ packages: dom-accessibility-api@0.6.3: resolution: {integrity: sha512-7ZgogeTnjuHbo+ct10G9Ffp0mif17idi0IyWNVA/wcwcm7NPOD/WEHVP3n7n3MhXqxoIYm8d6MuZohYWIZ4T3w==} - dompurify@3.4.15: - resolution: {integrity: sha512-EUBjM+B+lkDE41iE82DDSCfkoPGfXx8IxFxPMjNzm/Uk4xDet77rTN9wqlxlVg71kK7XGuUMv6wUxJUwwv+Xyw==} + dompurify@3.4.16: + resolution: {integrity: sha512-sqo+pNp3qRhCIpbgRi1y8Tgk27Bo2Ry7w0dC1NBeNTdZChWjz9Xb/KOoZbRP/R6pQZ80Qw8YhXw13hWWBbMRnQ==} dotenv@17.4.2: resolution: {integrity: sha512-nI4U3TottKAcAD9LLud4Cb7b2QztQMUEfHbvhTH09bqXTxnSie8WnjPALV/WMCrJZ6UV/qHJ6L03OqO3LcdYZw==} @@ -5863,6 +5879,12 @@ packages: exponential-backoff@3.1.3: resolution: {integrity: sha512-ZgEeZXj30q+I0EN+CbSSpIyPaJ5HVQD18Z1m+u1FXbAeT94mr1zw50q4q6jiiC447Nl/YTcIYSAftiGqetwXCA==} + express-rate-limit@8.6.1: + resolution: {integrity: sha512-0D493aP61w0TJ2A0wy27riRsO7FMQ7FK+KUHOKCSfPvYo0R55aiC6emCVgFUeShH0fq0ICPVzNcgoS+BsbXQCA==} + engines: {node: '>= 16'} + peerDependencies: + express: '>= 4.11' + express-rate-limit@8.7.0: resolution: {integrity: sha512-hOwV7WOxXfjRpAM1DSJWZDXx3GhplwD8IfwuwvogD8i1Qnkgosw/H45s4ZnFAUHDAhPjlY9hLBvJhKmGMyY26g==} engines: {node: '>= 16'} @@ -5937,10 +5959,6 @@ packages: file-uri-to-path@1.0.0: resolution: {integrity: sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==} - fill-range@7.1.1: - resolution: {integrity: sha512-YsGpe3WHLK8ZYi4tWDg2Jy3ebRz2rXowDxnld4bkQB00cc/1Zw9AWnC0i9ztDJitivtQvaI9KaLyKrc+hBW0yg==} - engines: {node: '>=8'} - finalhandler@1.1.2: resolution: {integrity: sha512-aAWcW57uxVNrQZqFXjITpW3sIUQmHGG3qSb9mUah9MgMC4NeWhNOlNjXEYq3HjRAvL6arUviZGGJsBg6z0zsWA==} engines: {node: '>= 0.8'} @@ -6317,10 +6335,6 @@ packages: resolution: {integrity: sha512-PhBY86zaxNZUuWP6h13Vu5oFe0XY6/UlKzQnYFELzGVHygP3MxmvTfYSG7GN3aIab/iWudSMgjSnG9Dq+nHrgA==} engines: {node: '>=16'} - is-number@7.0.0: - resolution: {integrity: sha512-41Cifkg6e8TylSpdtTpeLVMqvSBEVzTttHvERD741+pnZ8ANv0004MRL43QKPDlK9cGvNp6NZWZUBlbGXYxxng==} - engines: {node: '>=0.12.0'} - is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} @@ -7140,10 +7154,6 @@ packages: micromark@4.0.2: resolution: {integrity: sha512-zpe98Q6kvavpCr1NPVSCMebCKfD7CA2NqZ+rykeNhONIJBpc1tFKt9hucLGwha3jNTNI8lHpctWJWoimVF4PfA==} - micromatch@4.0.8: - resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} - engines: {node: '>=8.6'} - mime-db@1.52.0: resolution: {integrity: sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==} engines: {node: '>= 0.6'} @@ -8382,10 +8392,6 @@ packages: resolution: {integrity: sha512-WMi/OQ2axVTf/ykqCQgXiIct+mSQDFdH2fkwhPwgEwvJ1kSzZRiinb0zF2Xb8u4+OqPChmyI6MEu4EezNJz+FQ==} hasBin: true - to-regex-range@5.0.1: - resolution: {integrity: sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ==} - engines: {node: '>=8.0'} - toidentifier@1.0.1: resolution: {integrity: sha512-o5sSPKEkg/DIQNmH43V0/uerLrpzVedkUh8tGNvaeXpfpuwjKenlSox/2O/BTlZUtEe+JG7s5YhEz608PlAHRA==} engines: {node: '>=0.6'} @@ -12540,14 +12546,10 @@ snapshots: transitivePeerDependencies: - supports-color - brace-expansion@5.0.11: + brace-expansion@5.0.12: dependencies: balanced-match: 4.0.4 - braces@3.0.3: - dependencies: - fill-range: 7.1.1 - browserslist@4.29.0: dependencies: baseline-browser-mapping: 2.11.24 @@ -13066,7 +13068,7 @@ snapshots: dom-accessibility-api@0.6.3: {} - dompurify@3.4.15: + dompurify@3.4.16: optionalDependencies: '@types/trusted-types': 2.0.7 @@ -13334,6 +13336,14 @@ snapshots: exponential-backoff@3.1.3: {} + express-rate-limit@8.6.1(express@5.2.1): + dependencies: + debug: 4.4.3 + express: 5.2.1 + ip-address: 10.7.2 + transitivePeerDependencies: + - supports-color + express-rate-limit@8.7.0(express@5.2.1): dependencies: debug: 4.4.3 @@ -13427,10 +13437,6 @@ snapshots: file-uri-to-path@1.0.0: {} - fill-range@7.1.1: - dependencies: - to-regex-range: 5.0.1 - finalhandler@1.1.2: dependencies: debug: 2.6.9 @@ -13835,8 +13841,6 @@ snapshots: is-network-error@1.3.2: {} - is-number@7.0.0: {} - is-plain-obj@4.1.0: {} is-plain-object@2.0.4: @@ -14842,7 +14846,7 @@ snapshots: d3-sankey: 0.12.3 dagre-d3-es: 7.0.14 dayjs: 1.11.23 - dompurify: 3.4.15 + dompurify: 3.4.16 elkjs: 0.9.3 es-toolkit: 1.52.0 katex: 0.16.47 @@ -14898,7 +14902,7 @@ snapshots: lodash.throttle: 4.1.1 metro-resolver: 0.87.1 - metro-file-map@0.87.1: + metro-file-map@0.87.1(patch_hash=28741c2833798bbfda1432bd62f13f338d31f12a6161f4268a6cc6f01022018b): dependencies: debug: 4.4.3 fb-watchman: 2.0.2 @@ -14906,8 +14910,8 @@ snapshots: graceful-fs: 4.2.11 invariant: 2.2.4 jest-worker: 29.7.0 - micromatch: 4.0.8 nullthrows: 1.1.1 + picomatch: 2.3.2 transitivePeerDependencies: - supports-color @@ -15006,7 +15010,7 @@ snapshots: metro-cache-key: 0.87.1 metro-config: 0.87.1 metro-core: 0.87.1 - metro-file-map: 0.87.1 + metro-file-map: 0.87.1(patch_hash=28741c2833798bbfda1432bd62f13f338d31f12a6161f4268a6cc6f01022018b) metro-resolver: 0.87.1 metro-runtime: 0.87.1 metro-source-map: 0.87.1 @@ -15296,11 +15300,6 @@ snapshots: transitivePeerDependencies: - supports-color - micromatch@4.0.8: - dependencies: - braces: 3.0.3 - picomatch: 2.3.2 - mime-db@1.52.0: {} mime-db@1.54.0: {} @@ -15325,11 +15324,11 @@ snapshots: minimatch@10.2.6: dependencies: - brace-expansion: 5.0.11 + brace-expansion: 5.0.12 minimatch@9.0.9: dependencies: - brace-expansion: 5.0.11 + brace-expansion: 5.0.12 minimist@1.2.8: {} @@ -16660,10 +16659,6 @@ snapshots: dependencies: tldts-core: 6.1.86 - to-regex-range@5.0.1: - dependencies: - is-number: 7.0.0 - toidentifier@1.0.1: {} toml@4.2.0: {} diff --git a/pnpm-workspace.yaml b/pnpm-workspace.yaml index fe8f5d04d..edf5653d5 100644 --- a/pnpm-workspace.yaml +++ b/pnpm-workspace.yaml @@ -37,7 +37,7 @@ overrides: # readableEnded. Yamux 8.0.1 allows @libp2p/utils ^7.0.0 but the lock selected # 7.3.0, whose AbstractStream does not declare that field. 7.4.1 does. '@chainsafe/libp2p-yamux>@libp2p/utils': 7.4.1 - brace-expansion@<5.0.11: 5.0.11 + brace-expansion@<5.0.12: 5.0.12 # Socket.IO retains engine.io releases vulnerable to the protocol-revision # mismatch DoS. Select the first patched release; authsocket consumes the # same public API. @@ -61,9 +61,21 @@ overrides: # by GHSA-r5fr-rjxr-66jc (_.template imports key-name code injection). Select # the patched release already used elsewhere in the graph; the API is unchanged. lodash-es@<4.18.0: 4.18.1 + # Metro-file-map only uses micromatch.some, which delegates to Picomatch. + # The paired patch retains that matcher and removes the affected unused closure. + 'metro-file-map@0.87.1>micromatch': '-' strictDepBuilds: true trustPolicy: no-downgrade # Apply provenance downgrade checks to the recent supply-chain window. Older # packages predate consistent registry attestations and otherwise create known # false positives (for example chokidar 4.0.3 in pnpm's own documentation). trustPolicyIgnoreAfter: 525600 + +# Temporary exact-version build-tool repair; ownership/removal criteria live in +# governance/dependency-release-policy.json and the override exception registry. +packageExtensions: + 'metro-file-map@0.87.1': + dependencies: + picomatch: 2.3.2 +patchedDependencies: + 'metro-file-map@0.87.1': patches/metro-file-map@0.87.1.patch diff --git a/scripts/brc100-byte-boundary.test.mjs b/scripts/brc100-byte-boundary.test.mjs index 57202c76f..728f16950 100644 --- a/scripts/brc100-byte-boundary.test.mjs +++ b/scripts/brc100-byte-boundary.test.mjs @@ -103,7 +103,8 @@ test('the byte compatibility contract remains part of the public SDK wallet API' }) const dependentPackageSdkRanges = [ - ['packages/middleware/auth-express-middleware/package.json', '^2.8.5 || ^3.0.0'], + ['packages/middleware/auth-express-middleware/package.json', '^2.9.0 || ^3.0.0'], + ['packages/middleware/payment-express-middleware/package.json', '^2.9.0 || ^3.0.0'], ['packages/messaging/authsocket-client/package.json', '^2.4.1 || ^3.0.0'], ['packages/messaging/authsocket/package.json', '^2.4.1 || ^3.0.0'], ['packages/messaging/message-box-client/package.json', '^2.8.6 || ^3.0.0'], diff --git a/scripts/check-auth-express-consumers.mjs b/scripts/check-auth-express-consumers.mjs index 67024c876..7d1d5996c 100644 --- a/scripts/check-auth-express-consumers.mjs +++ b/scripts/check-auth-express-consumers.mjs @@ -134,6 +134,11 @@ try { } const challengeCount = challenges.length let paymentAttempted = false + let paymentAborted = false + // The candidate SDK requires prepare/submit/abort capabilities before it + // calls createAction. This probe refuses preparation, so nothing is spent + // or reserved and abortAction must remain unused. + wallet.abortAction = async () => { paymentAborted = true; return { aborted: true } } wallet.createAction = async () => { paymentAttempted = true; throw new Error('TEST_PAYMENT_DISABLED') } await assert.rejects(auth.fetch(base + '/paid'), /TEST_PAYMENT_DISABLED/) assert.equal(challenges.length, challengeCount + 1) @@ -142,6 +147,7 @@ try { assert.equal(challenge.headers['x-bsv-payment-satoshis-required'], '25') assert.ok(challenge.headers['x-bsv-auth-signature']) assert.equal(paymentAttempted, true) + assert.equal(paymentAborted, false) for (const config of [ {}, { headers: { 'content-type': 'application/json' } }, diff --git a/scripts/check-template-consumers.mjs b/scripts/check-template-consumers.mjs new file mode 100644 index 000000000..665b36aec --- /dev/null +++ b/scripts/check-template-consumers.mjs @@ -0,0 +1,225 @@ +#!/usr/bin/env node + +import assert from 'node:assert/strict' +import { createHash } from 'node:crypto' +import fs from 'node:fs/promises' +import os from 'node:os' +import path from 'node:path' +import process from 'node:process' +import { fileURLToPath } from 'node:url' + +import { createCommandRunner } from './lib/command-runner.mjs' + +const run = createCommandRunner({ timeoutMs: 180_000, maxBufferBytes: 20 * 1024 * 1024 }) +const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..') + +async function publishedSdkTarball(destination) { + const supplied = process.env.TEMPLATES_PUBLISHED_SDK_TARBALL + const filename = supplied ? path.resolve(supplied) : path.join(destination, 'bsv-sdk-2.8.0.tgz') + if (!supplied) { + // First-party packages already have an explicit workspace age exemption. + // npm's inherited global "before" cutoff cannot express that exception: + // use its package fetcher for the exact artifact, with scripts disabled, + // then verify the reviewed digest before any consumer installation. + await run( + 'npm', + [ + 'pack', + '--ignore-scripts', + '--json', + '--pack-destination', + destination, + 'https://registry.npmjs.org/@bsv/sdk/-/sdk-2.8.0.tgz' + ], + { cwd: destination } + ) + } + const bytes = await fs.readFile(filename) + assert.equal(bytes.length, 4_052_784, 'Published SDK artifact size changed') + assert.equal( + createHash('sha512').update(bytes).digest('base64'), + 'pXavnJa8F5ozKSOwVIphLZrTEJgEADlfj8Yu9CIOsdVC/X+CuGfHFwK9I5egr/EC1D2KCxmAbGls9erpc637Yw==', + 'Published SDK artifact must match its reviewed registry integrity' + ) + return filename +} + +// Executed in isolated consumers, so every class comes from the installed tarballs. +async function exercise(sdk, templates, check) { + const { Hash, LockingScript, OP, PrivateKey, ProtoWallet, PublicKey, Spend, Transaction } = sdk + const { MandalaAdmin, MandalaToken, MultiPushDrop, OpReturn, P2MSKH, R1K1Wallet } = templates + const privateKey = new PrivateKey(23) + const wallet = new ProtoWallet(privateKey) + const publicKeyHash = Hash.hash160(privateKey.toPublicKey().encode(true)) + const digest = script => Buffer.from(Hash.sha256(script.toBinary())).toString('hex') + const results = {} + + async function spend(name, lockingScript, unlocker) { + check.ok(lockingScript instanceof LockingScript) + const source = new Transaction() + source.addOutput({ lockingScript, satoshis: 1 }) + const transaction = new Transaction() + transaction.addInput({ sourceTransaction: source, sourceOutputIndex: 0, sequence: 0xffffffff }) + transaction.addOutput({ lockingScript: new LockingScript([{ op: OP.OP_TRUE }]), satoshis: 1 }) + const unlockingScript = await unlocker.sign(transaction, 0) + check.equal( + new Spend({ + sourceTXID: source.id('hex'), + sourceOutputIndex: 0, + sourceSatoshis: 1, + lockingScript, + transactionVersion: transaction.version, + otherInputs: [], + inputIndex: 0, + unlockingScript, + outputs: transaction.outputs, + inputSequence: 0xffffffff, + lockTime: transaction.lockTime + }).validate(), + true, + `${name} signature must execute against its source contract` + ) + results[name] = digest(lockingScript) + } + + const assetId = `${'ab'.repeat(32)}.0` + const token = new MandalaToken() + await spend('MandalaToken', token.lock(assetId, 1, publicKeyHash), token.unlock(privateKey)) + check.throws(() => token.lock(assetId, 0, publicKeyHash), /positive/) + const adminData = { kind: 'issue', assetId, amount: 1 } + await spend( + 'MandalaAdmin', + await MandalaAdmin.lock({ wallet, data: adminData }), + MandalaAdmin.unlock({ wallet, data: adminData }) + ) + const pushDrop = new MultiPushDrop(wallet) + const protocolID = [1, 'packed template consumer'] + await spend( + 'MultiPushDrop', + await pushDrop.lock([[1, 2, 3]], protocolID, 'fixture', ['self']), + pushDrop.unlock(protocolID, 'fixture', 'self') + ) + const counterparty = privateKey.toPublicKey().toString() + const { publicKey } = await wallet.getPublicKey({ + protocolID: [1, 'multi sig brc29'], + keyID: 'fixture', + counterparty, + forSelf: true + }) + const pubkeys = [publicKey, new PrivateKey(29).toPublicKey().toString()] + const multisig = new P2MSKH() + await spend( + 'P2MSKH', + multisig.lock( + undefined, + pubkeys.map(key => PublicKey.fromString(key)), + 1 + ), + multisig.unlock(wallet, { pubkeys, keyID: 'fixture', counterparty }) + ) + const recovery = new R1K1Wallet() + await spend( + 'R1K1Wallet', + await recovery.lock( + Array.from({ length: 20 }, () => 7), + publicKeyHash + ), + recovery.unlock({ path: 'k1', privateKey }) + ) + const opReturn = new OpReturn().lock(['packed', 'consumer']) + check.deepEqual(OpReturn.decode(opReturn), ['packed', 'consumer']) + results.OpReturn = digest(opReturn) + return results +} + +async function pack(directory, destination) { + const { stdout } = await run('pnpm', ['pack', '--json', '--pack-destination', destination], { + cwd: directory, + env: { ...process.env, npm_config_ignore_scripts: 'true' } + }) + return path.resolve(JSON.parse(stdout).filename) +} + +function consumerSource(format) { + const imports = + format === 'cjs' + ? `const sdk = require('@bsv/sdk') +const templates = require('@bsv/templates') +const { MandalaToken } = require('@bsv/templates/MandalaToken.ts') +const check = require('node:assert/strict')` + : `import * as sdk from '@bsv/sdk' +import * as templates from '@bsv/templates' +import { MandalaToken } from '@bsv/templates/MandalaToken.ts' +import check from 'node:assert/strict'` + return `${imports} +check.equal(MandalaToken, templates.MandalaToken) +;(${exercise.toString()})(sdk, templates, check).then(result => console.log(JSON.stringify(result))) +` +} + +async function completeConsumers(pending) { + // Drain every started child before the caller removes their temporary directories. + const settled = await Promise.allSettled(pending) + return settled.map(result => { + if (result.status === 'rejected') throw result.reason + return result.value + }) +} + +const temporary = await fs.mkdtemp(path.join(os.tmpdir(), 'template-consumers-')) +try { + const templates = await pack(path.join(root, 'packages/helpers/ts-templates'), temporary) + const candidate = await pack(path.join(root, 'packages/sdk'), temporary) + const sdkManifest = JSON.parse( + await fs.readFile(path.join(root, 'packages/sdk/package.json'), 'utf8') + ) + const profiles = [ + { label: 'candidate', sdk: candidate, version: sdkManifest.version }, + { + label: 'published', + sdk: await publishedSdkTarball(temporary), + version: '2.8.0' + } + ] + // Profiles install into separate directories; formats only read their shared dependencies. + const profileResults = await completeConsumers( + profiles.map(async profile => { + const cwd = path.join(temporary, profile.label) + await fs.mkdir(cwd) + await fs.writeFile(path.join(cwd, 'package.json'), '{"private":true,"type":"module"}\n') + await run( + 'npm', + [ + 'install', + '--ignore-scripts', + '--no-audit', + '--no-fund', + '--package-lock=false', + '--omit=dev', + templates, + profile.sdk + ], + { cwd } + ) + const installed = JSON.parse( + await fs.readFile(path.join(cwd, 'node_modules/@bsv/sdk/package.json'), 'utf8') + ) + assert.equal(installed.version, profile.version) + return await completeConsumers( + ['cjs', 'mjs'].map(async format => { + const filename = `consumer.${format}` + await fs.writeFile(path.join(cwd, filename), consumerSource(format)) + const { stdout } = await run(process.execPath, [filename], { cwd }) + console.log( + `Verified packed templates script construction and signing: SDK ${profile.version}, ${format}.` + ) + return JSON.parse(stdout) + }) + ) + }) + ) + const results = profileResults.flat() + for (const result of results.slice(1)) assert.deepEqual(result, results[0]) +} finally { + await fs.rm(temporary, { recursive: true, force: true }) +} diff --git a/scripts/check-wallet-toolbox-platform.mjs b/scripts/check-wallet-toolbox-platform.mjs index 2b7ed9223..bff2ab56b 100644 --- a/scripts/check-wallet-toolbox-platform.mjs +++ b/scripts/check-wallet-toolbox-platform.mjs @@ -9,6 +9,7 @@ import process from 'node:process' import { fileURLToPath, pathToFileURL } from 'node:url' import { createCommandRunner } from './lib/command-runner.mjs' +import { assertMetroWatcherContract } from './lib/metro-watcher-contract.mjs' const COMMAND_TIMEOUT_MS = 240_000 const MAX_BUFFER_BYTES = 30 * 1024 * 1024 @@ -410,6 +411,10 @@ function hermesCompilerPath() { } async function checkMobile(consumerDirectory, budget) { + console.log( + 'Verified Metro watcher contract:', + assertMetroWatcherContract(toolResolver('metro/package.json')) + ) const entryPath = path.join(consumerDirectory, 'index.js') await fs.writeFile(entryPath, consumerEntry(expectedPackage)) await fs.writeFile( @@ -487,9 +492,24 @@ async function checkMobile(consumerDirectory, budget) { const hermesPath = hermesCompilerPath() const bytecodePath = path.join(consumerDirectory, 'wallet-toolbox-mobile.hbc') - await run(hermesPath, ['-O', '-emit-binary', '-out', bytecodePath, bundlePath]) - const hermesBytecode = await fs.readFile(bytecodePath) + // Hermes retains its input filename in bytecode. A random absolute temporary + // path makes identical bundles cross compression budgets nondeterministically. + // Keep source/debug data and compile from a stable relative filename instead. + const compile = async directory => { + await run( + hermesPath, + ['-O', '-emit-binary', '-out', path.basename(bytecodePath), path.basename(bundlePath)], + { cwd: directory } + ) + return fs.readFile(path.join(directory, path.basename(bytecodePath))) + } + const hermesBytecode = await compile(consumerDirectory) if (hermesBytecode.length === 0) throw new Error('Hermes compiler emitted empty bytecode') + const repeatDirectory = await fs.mkdtemp(path.join(consumerDirectory, 'hermes-repro-')) + await fs.copyFile(bundlePath, path.join(repeatDirectory, path.basename(bundlePath))) + if (!hermesBytecode.equals(await compile(repeatDirectory))) { + throw new Error('Hermes bytecode must reproduce across independent build directories') + } const hermesSizes = sizes(hermesBytecode) validateBudget(hermesSizes, budget.hermes, 'Hermes mobile bytecode') diff --git a/scripts/ci-integration/mutation-workflows.test.mjs b/scripts/ci-integration/mutation-workflows.test.mjs index 020351933..0e246f72d 100644 --- a/scripts/ci-integration/mutation-workflows.test.mjs +++ b/scripts/ci-integration/mutation-workflows.test.mjs @@ -158,6 +158,7 @@ test('full campaign receipts cannot borrow another attempt or a partial manual t 'wallet-snapshot-sync', 'wallet-snapshot-sync-destination', 'wallet-snapshot-sync-rows', + 'wallet-snapshot-remote-reader', 'root-eviction-journal', 'root-eviction-records' ]) diff --git a/scripts/ci-orchestration.test.mjs b/scripts/ci-orchestration.test.mjs index 6a861a5b4..b9d03c047 100644 --- a/scripts/ci-orchestration.test.mjs +++ b/scripts/ci-orchestration.test.mjs @@ -5,8 +5,11 @@ import { join } from 'node:path' import test from 'node:test' import { REPOSITORY_ROOT } from './repository-health.mjs' +import { buildMutationTargets } from '../governance/mutation-testing/targets.mjs' +import { partitionedMutationTargets, partitionMutationTarget } from './mutation-partitions.mjs' const CI_PATH = join(REPOSITORY_ROOT, '.github/workflows/ci.yml') +const MUTATION_PATH = join(REPOSITORY_ROOT, '.github/workflows/mutation-tests.yml') const CONFORMANCE_PATH = join(REPOSITORY_ROOT, '.github/workflows/conformance.yml') const RUNTIME_PATH = join(REPOSITORY_ROOT, '.github/workflows/container-runtime-contract.yml') const WALLET_MOBILE_COVERAGE_PATH = join( @@ -24,6 +27,50 @@ function workflowJobBlocks(workflow) { })) } +function assertWalletMutationTimeout(job, defaultMinutes) { + const targets = + '["revenue-lineage-package","revenue-lineage-graph","sdk-revenue-listing-funding","output-lookup-session-records","output-lookup-session-payloads","wallet-recovery-codec","wallet-recovery-installation","wallet-recovery-store","wallet-funding-store","wallet-recovery-transitions","wallet-recovery-controller","root-eviction-storage","root-eviction-journal","root-eviction-records","wallet-retained-snapshot","wallet-snapshot-journal","wallet-snapshot-sync","wallet-snapshot-sync-destination","wallet-snapshot-sync-rows","wallet-snapshot-archive","wallet-snapshot-remote-http","wallet-snapshot-remote-reader","wallet-snapshot-remote-service"]' + const expected = ` timeout-minutes: \${{ contains(fromJSON('${targets}'), matrix.target) && 90 || ${defaultMinutes} }}` + assert.equal(job.source.match(/^ {4}timeout-minutes: .+$/m)?.[0], expected) +} + +test('CI downloads every canonical execution partition before verifying its complete union', () => { + const targets = buildMutationTargets(REPOSITORY_ROOT) + const partitioned = partitionedMutationTargets(Object.keys(targets), targets) + assert.ok(partitioned.length > 0) + assert.ok(partitioned.includes('wallet-snapshot-sync')) + assert.deepEqual( + partitionMutationTarget('wallet-snapshot-sync', targets['wallet-snapshot-sync']).map( + part => part.id + ), + ['session', 'checkpoint', 'copy', 'storage', 'primary'] + ) + const workflow = readFileSync(CI_PATH, 'utf8') + const gate = workflowJobBlocks(workflow).find(job => job.name === 'mutation-quality').source + const verify = gate.indexOf( + ' - name: Require every selected canonical partition target gate' + ) + assert.ok(verify > 0) + const downloads = gate + .slice(0, verify) + .split(/^ {6}- /m) + .filter(step => step.startsWith('uses: actions/download-artifact@')) + for (const target of partitioned) { + const selected = downloads.filter(step => + step.includes(` path: .mutation-parts/${target}\n`) + ) + assert.equal(selected.length, 1, `${target} requires exactly one artifact download`) + assert.ok( + selected[0].includes( + ` if: contains(fromJSON(needs.prepare.outputs.partition-targets || '[]'), '${target}')\n` + ) + ) + assert.ok(selected[0].includes(` pattern: mutation-${target}-*\n`)) + assert.match(selected[0], /^uses: actions\/download-artifact@[a-f0-9]{40} /) + } + assert.match(gate.slice(verify), /--target "\$target" --directory "\.mutation-parts\/\$target"/) +}) + test('CI shares one audited build across coverage and browser consumer lanes', () => { const workflow = readFileSync(CI_PATH, 'utf8') @@ -123,10 +170,7 @@ test('CI bounds every job and allocates no runner for an empty infrastructure ma assert.ok(jobs.length > 0) for (const job of jobs) { if (job.name === 'mutation-tests') { - assert.match( - job.source, - /^ timeout-minutes: \$\{\{ contains\(fromJSON\('[^']+'\), matrix\.target\) && 90 \|\| 45 \}\}$/m - ) + assertWalletMutationTimeout(job, 45) } else { assert.match(job.source, /^ timeout-minutes: \d+$/m, `${job.name} must have a timeout`) } @@ -139,6 +183,25 @@ test('CI bounds every job and allocates no runner for an empty infrastructure ma assert.match(workflow, /\( "\$INFRA_RESULT" != "success" && "\$INFRA_RESULT" != "skipped" \)/) }) +test('wallet mutation allowances preserve other limits and complete campaign execution', () => { + for (const [path, defaultMinutes, maxParallel] of [ + [CI_PATH, 45, 6], + [MUTATION_PATH, 45, 20] + ]) { + const job = workflowJobBlocks(readFileSync(path, 'utf8')).find( + job => job.name === 'mutation-tests' + ) + assert.ok(job, path) + assertWalletMutationTimeout(job, defaultMinutes) + assert.match(job.source, /^ {6}fail-fast: false$/m) + assert.match(job.source, new RegExp(`^ max-parallel: ${maxParallel}$`, 'm')) + assert.match( + job.source, + /^ {8}run: node scripts\/mutation-testing\.mjs --target "\$\{\{ matrix\.target \}\}" --partition "\$\{\{ matrix\.partition \}\}"$/m + ) + } +}) + test('specialized workflows are bounded and required conformance checks always run on PRs', () => { const conformance = readFileSync(CONFORMANCE_PATH, 'utf8') const runtime = readFileSync(RUNTIME_PATH, 'utf8') @@ -351,6 +414,77 @@ test('every HTTP latency scenario retains its own required coverage execution', assert.doesNotMatch(wallet, /continue-on-error|passWithNoTests/) }) +function nativeWalletFixtureSteps() { + const wallet = workflowJobBlocks(readFileSync(CI_PATH, 'utf8')).find( + job => job.name === 'coverage-wallet' + ).source + const matches = [...wallet.matchAll(/^ {6}- /gm)] + const steps = matches.map((match, index) => + wallet.slice(match.index, matches[index + 1]?.index ?? wallet.length) + ) + return { wallet, steps } +} + +function onlyFixtureStep(steps, command) { + const matches = steps.filter(step => step.split('\n').includes(` run: ${command}`)) + assert.equal(matches.length, 1) + const step = matches[0] + assert.match(step, /^ {8}if: matrix.id == 'shard-1'$/m) + assert.match(step, /^ {8}working-directory: packages\/wallet\/wallet-toolbox$/m) + assert.doesNotMatch(step, /continue-on-error/) + return steps.indexOf(step) +} + +function assertNativeWalletJob(wallet) { + assert.match(wallet, /^ {4}needs: prepare$/m) + assert.match(wallet, /^ {4}timeout-minutes: 40$/m) + assert.match(wallet, /^ {4}permissions:\n {6}contents: read\n {4}strategy:/m) + assert.doesNotMatch(wallet, /continue-on-error/) + // This gate runs before dependency installation; validate the governed native + // job's explicit matrix and service block without loading a workspace parser. + assert.match( + wallet, + /^ {4}strategy:\n {6}fail-fast: false\n {6}matrix:\n {8}include:\n {10}- \{ id: shard-1, shard: 1 \}\n {10}- \{ id: shard-2, shard: 2 \}\n {10}- \{ id: shard-3, shard: 3 \}\n {10}- \{ id: shard-4, shard: 4 \}\n {10}- \{ id: sync-http-0, latency: 0 \}\n {10}- \{ id: sync-http-1000, latency: 1000 \}\n {4}services:$/m + ) + assert.match( + wallet, + /^ {6}postgres:\n {8}image: postgres@sha256:d5daad18926b71c3d663f358af0aea798670cb79fb550c106d19662a9d1627ef(?: #[^\n]*)?$/m + ) + assert.match(wallet, /^ {8}ports:\n {10}- 5432:5432\n {8}options: >-$/m) +} + +test('native snapshot process-loss proof uses the same-head build in exactly one required wallet shard', () => { + const { wallet, steps } = nativeWalletFixtureSteps() + const proof = onlyFixtureStep(steps, 'node test/storage/snapshotArchiveCrash.cjs') + const restored = steps.findIndex(step => + step.includes('run: tar --extract --gzip --file .ci-artifacts/build-outputs.tar.gz') + ) + const coverage = steps.findIndex(step => + step.includes('name: Generate wallet-toolbox coverage shard') + ) + assert.ok(restored >= 0 && restored < proof && proof < coverage) + assertNativeWalletJob(wallet) +}) + +test('native MySQL uses a bounded pinned fixture in the existing required wallet shard', () => { + const { wallet, steps } = nativeWalletFixtureSteps() + const provision = onlyFixtureStep( + steps, + 'node test/storage/snapshotArchiveDocker.cjs provision-hosted-image' + ) + const proof = onlyFixtureStep(steps, 'node test/storage/runSnapshotArchiveMysql.cjs') + for (const index of [provision, proof]) { + assert.match(steps[index], /^ {8}timeout-minutes: 5$/m) + assert.match(steps[index], /^ {8}env:\n {10}TS_STACK_SNAPSHOT_HOSTED_MYSQL: '1'\n {8}run:/m) + } + const sqlite = onlyFixtureStep(steps, 'node test/storage/snapshotArchiveCrash.cjs') + const coverage = steps.findIndex(step => + step.includes('name: Generate wallet-toolbox coverage shard') + ) + assert.ok(sqlite < provision && provision < proof && proof < coverage) + assertNativeWalletJob(wallet) +}) + test('the mutation quality job accepts skipped execution only for explicitly empty scope', () => { const job = workflowJobBlocks(readFileSync(CI_PATH, 'utf8')).find( candidate => candidate.name === 'mutation-quality' @@ -387,3 +521,38 @@ test('the mutation quality job accepts skipped execution only for explicitly emp }) assert.notEqual(failedBuild.status, 0) }) + +test('journal native recovery remains mandatory alongside all archive fixture families', () => { + const { wallet, steps } = nativeWalletFixtureSteps() + const archive = onlyFixtureStep(steps, 'node test/storage/runSnapshotArchiveMysql.cjs') + const sqlite = onlyFixtureStep(steps, 'node test/storage/snapshotJournalSqliteCrash.cjs') + const mysql = onlyFixtureStep(steps, 'node test/storage/runSnapshotJournalMysql.cjs') + const coverage = steps.findIndex(step => + step.includes('name: Generate wallet-toolbox coverage shard') + ) + assert.match(steps[sqlite], /^ {8}timeout-minutes: 3$/m) + assert.match(steps[mysql], /^ {8}timeout-minutes: 10$/m) + assert.ok(steps[mysql].includes("TS_STACK_SNAPSHOT_HOSTED_MYSQL: '1'")) + assert.ok(archive < sqlite && sqlite < mysql && mysql < coverage) + assertNativeWalletJob(wallet) +}) + +test('actual pinned inventory runs as a required installed-tool regression before build reuse', () => { + const jobs = workflowJobBlocks(readFileSync(CI_PATH, 'utf8')) + const health = jobs.find(job => job.name === 'repository-health').source + const prepare = jobs.find(job => job.name === 'prepare').source + assert.ok(health.includes('node --test scripts/*.test.mjs')) + assert.ok(!health.includes('mutation-partitions-engine.integration.mjs')) + const install = prepare.indexOf('run: pnpm install --frozen-lockfile --ignore-scripts') + const verify = prepare.indexOf( + 'run: node --test scripts/mutation-partitions-engine.integration.mjs' + ) + const build = prepare.indexOf('- name: Build workspace') + assert.ok(install >= 0 && install < verify && verify < build) + const steps = prepare.split(/^ {6}- /m) + const selected = steps.filter(step => + step.includes('run: node --test scripts/mutation-partitions-engine.integration.mjs') + ) + assert.equal(selected.length, 1) + assert.ok(!selected[0].includes('if:') && !selected[0].includes('continue-on-error')) +}) diff --git a/scripts/codeql-config.test.mjs b/scripts/codeql-config.test.mjs index 491536645..a9555cff1 100644 --- a/scripts/codeql-config.test.mjs +++ b/scripts/codeql-config.test.mjs @@ -63,11 +63,14 @@ test('advanced CodeQL preserves authored languages, events, permissions, and req .filter(Boolean) .filter(path => !generatedBoundaries.some(boundary => matchesGeneratedBoundary(path, boundary))) - assert.deepEqual(readIndentedList(workflow, 'language', 8), ['actions', 'javascript-typescript']) - assert.deepEqual( - authoredPythonFiles, - [], - 'authored Python requires restoring the Python CodeQL lane' + assert.deepEqual(readIndentedList(workflow, 'language', 8), [ + 'actions', + 'javascript-typescript', + ...(authoredPythonFiles.length > 0 ? ['python'] : []) + ]) + assert.ok( + authoredPythonFiles.includes('conformance/runner/scripts/brc118-vectors.py'), + 'the independent BRC-118 oracle must remain inside the authored CodeQL boundary' ) assert.match(workflow, /^ push:\n branches: \[main\]$/m) assert.match(workflow, /^ pull_request:\n branches: \[main\]$/m) diff --git a/scripts/dependency-release-governance.test.mjs b/scripts/dependency-release-governance.test.mjs index 12dd494d6..06fb9c540 100644 --- a/scripts/dependency-release-governance.test.mjs +++ b/scripts/dependency-release-governance.test.mjs @@ -22,13 +22,21 @@ test('dependency and release governance is internally complete', () => { assert.deepEqual(validateDependencyReleaseGovernance(), []) const overrides = collectOverrides() - assert.equal(overrides.length, 26) + assert.equal(overrides.length, 30) + assert.deepEqual( + overrides.filter(entry => entry.selector === 'nodemon'), + ['uhrp-server-basic', 'uhrp-server-cloud-bucket', 'wab'].map(component => ({ + source: `infra/${component}/package.json`, + selector: 'nodemon', + value: { chokidar: '4.0.3' } + })) + ) assert.equal(overrides.filter(entry => entry.selector === 'gaxios').length, 8) assert.equal(overrides.filter(entry => entry.selector === 'uuid').length, 3) assert.equal(overrides.filter(entry => entry.selector === 'brace-expansion').length, 4) assert.equal( - overrides.find(entry => entry.selector === 'brace-expansion@<5.0.11')?.value, - '5.0.11' + overrides.find(entry => entry.selector === 'brace-expansion@<5.0.12')?.value, + '5.0.12' ) assert.equal(overrides.find(entry => entry.selector === 'engine.io@<6.6.10')?.value, '6.6.10') assert.equal(overrides.filter(entry => entry.selector === 'toml@<4.2.0').length, 1) @@ -36,6 +44,10 @@ test('dependency and release governance is internally complete', () => { assert.equal(overrides.filter(entry => entry.selector === 'js-yaml').length, 1) assert.equal(overrides.find(entry => entry.selector === 'lodash-es@<4.18.0')?.value, '4.18.1') assert.equal(overrides.filter(entry => entry.selector.includes('image-size')).length, 0) + assert.equal( + overrides.find(entry => entry.selector === 'metro-file-map@0.87.1>micromatch')?.value, + "'-'" + ) }) test('pnpm override parsing preserves scoped parent selectors', () => { diff --git a/scripts/lib/fixtures/metro-watcher-contract.json b/scripts/lib/fixtures/metro-watcher-contract.json new file mode 100644 index 000000000..661be712c --- /dev/null +++ b/scripts/lib/fixtures/metro-watcher-contract.json @@ -0,0 +1,1702 @@ +{ + "oracle": "Unmodified published metro-file-map0.87.1 with micromatch4.0.8/Picomatch2.3.2", + "oracleSourceSha256": "d962d60bf25b7bd12a7679894fbb0c7e9d646b4efb1ab4a97db6f13c62eb8722", + "files": [ + "mod.ts", + "src/mod.ts", + "src/mod.tsx", + "src/sub/file.js", + ".hidden/mod.ts", + "src/.hidden/mod.ts", + "src/generated/mod.ts", + "src/test.generated.ts", + "src/B.ts", + "src/a.ts", + "src/sub/file", + "src\\mod.ts", + "", + "/src/mod.ts" + ], + "cases": [ + { + "type": "f", + "dot": false, + "globs": [], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + false, + false, + false, + false, + true, + true, + true, + true, + false, + true, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + false, + false, + false, + true, + true, + true, + true, + false, + true, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + false, + false, + false, + false, + true, + true, + false, + true, + false, + true, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true + ] + }, + { + "type": "f", + "dot": false, + "globs": ["**/.hidden/**"], + "expected": [ + false, + false, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + { + "type": "f", + "dot": false, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + false, + false, + false, + false, + true, + false, + false, + false, + false, + false, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": [], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + false, + true, + true, + true, + true, + true, + true, + false, + true, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + false, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + false, + false, + true, + true, + true, + true, + false, + true, + false, + true, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + false, + false, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true + ] + }, + { + "type": "f", + "dot": true, + "globs": ["**/.hidden/**"], + "expected": [ + false, + false, + false, + false, + true, + true, + false, + false, + false, + false, + false, + false, + false, + false + ] + }, + { + "type": "f", + "dot": true, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + false, + false, + true, + true, + true, + false, + false, + false, + false, + false, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": [], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["**/.hidden/**"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": false, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": [], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["**/.hidden/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "d", + "dot": true, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": [], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["**/.hidden/**"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": false, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": [], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["**/.hidden/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": "l", + "dot": true, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": false, + "globs": [], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["**/.hidden/**"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": false, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + true, + true, + false, + false, + true, + true, + true, + true, + true, + true, + false, + true + ] + }, + { + "type": null, + "dot": true, + "globs": [], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["**/*"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["**/*.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["**/*.{ts,tsx}"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["!**/generated/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["src/**", "!**/*.generated.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["**/[a-z]*.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["**/+(index|mod).ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["**/.hidden/**"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + }, + { + "type": null, + "dot": true, + "globs": ["*.js", "**/mod.ts"], + "expected": [ + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true, + true + ] + } + ] +} diff --git a/scripts/lib/metro-watcher-contract.mjs b/scripts/lib/metro-watcher-contract.mjs new file mode 100644 index 000000000..0366a511c --- /dev/null +++ b/scripts/lib/metro-watcher-contract.mjs @@ -0,0 +1,59 @@ +import assert from 'node:assert/strict' +import fs from 'node:fs' +import { createRequire } from 'node:module' +import path from 'node:path' + +/** Check the real installed watcher against an immutable published oracle. + * The ordinary inputs include negative arrays, brace alternatives, extglobs, + * dot files, separators and every watcher file type. No advisory payloads. */ +export function assertMetroWatcherContract(metroPackagePath) { + const metroRequire = createRequire(metroPackagePath) + const commonPath = path.join( + path.dirname(metroRequire.resolve('metro-file-map/package.json')), + 'src/watchers/common.js' + ) + const watcherRequire = createRequire(commonPath) + const common = watcherRequire(commonPath) + const fixture = JSON.parse( + fs.readFileSync(new URL('./fixtures/metro-watcher-contract.json', import.meta.url), 'utf8') + ) + let checked = 0 + for (const { type, dot, globs, expected } of fixture.cases) { + for (const [index, file] of fixture.files.entries()) { + assert.equal( + common.includedByGlob(type, globs, dot, file), + expected[index], + `Metro watcher compatibility: ${JSON.stringify({ type, dot, globs, file })}` + ) + checked++ + } + } + assert.equal(checked, 1120) + for (const [name, value] of Object.entries({ + DELETE_EVENT: 'delete', + TOUCH_EVENT: 'touch', + RECRAWL_EVENT: 'recrawl', + ALL_EVENT: 'all' + })) + assert.equal(common[name], value) + for (const [kind, expected] of [ + ['isSymbolicLink', 'l'], + ['isDirectory', 'd'], + ['isFile', 'f'] + ]) { + const stat = Object.fromEntries( + ['isSymbolicLink', 'isDirectory', 'isFile'].map(method => [method, () => method === kind]) + ) + assert.equal(common.typeFromStat(stat), expected) + } + for (const file of ['a/b.ts', String.raw`a\b.ts`]) { + assert.equal(common.posixPathMatchesPattern(/\.ts$/, file), true) + assert.equal(common.posixPathMatchesPattern(/\.js$/, file), false) + } + assert.equal(common.posixPathMatchesPattern(/^a\/b\.ts$/, path.join('a', 'b.ts')), true) + assert.equal(watcherRequire('picomatch/package.json').version, '2.3.2') + for (const name of ['micromatch', 'braces']) { + assert.throws(() => watcherRequire.resolve(name), { code: 'MODULE_NOT_FOUND' }) + } + return { publishedOracle: fixture.oracle, watcherCases: checked, removedDependencyClosure: true } +} diff --git a/scripts/mutation-partitions-engine.integration.mjs b/scripts/mutation-partitions-engine.integration.mjs new file mode 100644 index 000000000..1316543f7 --- /dev/null +++ b/scripts/mutation-partitions-engine.integration.mjs @@ -0,0 +1,77 @@ +import assert from 'node:assert/strict' +import test from 'node:test' +import { join } from 'node:path' +import { REPOSITORY_ROOT } from './mutation-testing.mjs' +import { buildMutationTargets } from '../governance/mutation-testing/targets.mjs' +import { canonicalInventory, targetSources } from './mutation-final-qualification.mjs' +import { + partitionMutationTarget, + selectedMutationPartition, + mutationExecutionMatrix +} from './mutation-partitions.mjs' + +function lexicalCompare(left, right) { + if (left < right) return -1 + if (left > right) return 1 + return 0 +} + +async function* inventories(directory, parts) { + for (const part of parts) + yield canonicalInventory( + directory, + targetSources(REPOSITORY_ROOT, part.target), + part.target.mutate + ) +} + +test('primary sync execution retains every original file/range, full configuration and future helper', async () => { + const canonical = buildMutationTargets(REPOSITORY_ROOT)['wallet-snapshot-sync'] + const parts = partitionMutationTarget('wallet-snapshot-sync', canonical) + assert.deepEqual( + parts.map(part => part.id), + ['session', 'checkpoint', 'copy', 'storage', 'primary'] + ) + assert.deepEqual( + parts.flatMap(part => part.target.mutate).toSorted(lexicalCompare), + [...canonical.mutate].toSorted(lexicalCompare) + ) + const owners = new Map() + const tuples = [] + const directory = join(REPOSITORY_ROOT, canonical.packageDirectory) + const tuple = mutant => + JSON.stringify({ + file: mutant.fileName, + name: mutant.mutatorName, + location: mutant.location, + replacement: mutant.replacement + }) + for (const part of parts) { + assert.deepEqual(part.target, { ...canonical, mutate: part.target.mutate }) + assert.equal(part.target.runnerOptions, canonical.runnerOptions) + for (const specification of part.target.mutate) { + const file = specification.replace(/:\d+(?:-\d+)?$/, '') + assert.ok(!owners.has(file) || owners.get(file) === part.id) + owners.set(file, part.id) + } + } + for await (const mutants of inventories(directory, parts)) tuples.push(...mutants.map(tuple)) + const original = ( + await canonicalInventory(directory, targetSources(REPOSITORY_ROOT, canonical), canonical.mutate) + ).map(tuple) + assert.ok(original.length > 0) + assert.equal(new Set(tuples).size, tuples.length) + assert.deepEqual(tuples.toSorted(lexicalCompare), original.toSorted(lexicalCompare)) + assert.equal(owners.get('src/storage/WalletStorageManager.ts'), 'primary') + assert.equal(owners.get('src/storage/StorageKnex.ts'), 'storage') + assert.equal(owners.get('src/storage/snapshot/SnapshotSync.ts'), 'session') + const helper = 'src/storage/snapshot/FuturePrimaryHelper.ts' + const future = partitionMutationTarget('wallet-snapshot-sync', { + ...canonical, + mutate: [...canonical.mutate, helper] + }) + assert.deepEqual(future[0].target.mutate, [...parts[0].target.mutate, helper]) + assert.equal(selectedMutationPartition('wallet-snapshot-sync', canonical), canonical) + const targets = buildMutationTargets(REPOSITORY_ROOT) + assert.ok(mutationExecutionMatrix(Object.keys(targets), targets).include.length <= 256) +}) diff --git a/scripts/mutation-partitions.mjs b/scripts/mutation-partitions.mjs index 1a63da316..e98ac9811 100644 --- a/scripts/mutation-partitions.mjs +++ b/scripts/mutation-partitions.mjs @@ -12,16 +12,125 @@ const plans = new Map([ ]) } ], + [ + 'wallet-snapshot-sync', + { + fallback: 'session', + files: new Map([ + ['src/utility/runInSeries.ts', 'session'], + ['src/storage/sync/syncSession.ts', 'session'], + ['src/storage/snapshot/SnapshotSync.ts', 'session'], + ['src/storage/sync/syncCheckpoint.ts', 'checkpoint'], + ['src/storage/snapshot/runSnapshotSyncSession.ts', 'copy'], + ['src/storage/StorageKnex.ts', 'storage'], + ['src/storage/WalletStorageManager.ts', 'primary'] + ]) + } + ], [ 'wallet-retained-snapshot', { fallback: 'lifecycle', files: new Map([ ['src/storage/snapshot/KnexWalletReadSnapshot.ts', 'reader'], + ['src/storage/schema/snapshotProfileIndexMigration.ts', 'profile-index'], + ['src/storage/schema/snapshotRelationIndexMigration.ts', 'relation-index'], + ['src/storage/schema/snapshotCertificateIndexMigration.ts', 'certificate-index'], + ['src/storage/schema/snapshotGlobalIndexMigration.ts', 'global-index'], + ['src/storage/schema/snapshotGlobalIndexModel.ts', 'global-index'], + ['src/storage/schema/snapshotGlobalIndexMysql.ts', 'global-mysql'], + ['src/storage/schema/snapshotGlobalIndexSqlite.ts', 'global-sqlite'], + ['src/storage/schema/snapshotGlobalIndexBootstrap.ts', 'global-bootstrap'], + ['src/storage/schema/snapshotGlobalIndexTriggers.ts', 'global-triggers'], + ['src/storage/schema/snapshotSqliteIdentity.ts', 'sqlite-identity'], + ['src/storage/schema/snapshotSqliteMembership.ts', 'sqlite-membership'], + ['src/storage/schema/snapshotSqliteIndexGeneration.ts', 'sqlite-generation'], + ['src/storage/schema/snapshotSqliteIndexBootstrap.ts', 'sqlite-bootstrap'], + ['src/storage/schema/snapshotSqliteIndexState.ts', 'sqlite-bootstrap'], + ['src/storage/schema/snapshotSqliteIndexRetirement.ts', 'sqlite-retirement'], + ['src/storage/schema/snapshotSqliteLegacyOwnership.ts', 'sqlite-generation'], + ['src/storage/schema/snapshotSqliteIndexMigration.ts', 'sqlite-retirement'], ['src/storage/StorageKnex.ts', 'storage'], ['src/storage/StorageProvider.ts', 'storage'] ]) } + ], + [ + 'wallet-snapshot-journal', + { + fallback: 'revision', + files: new Map([ + ['src/storage/snapshot/journal/SnapshotJournalRevision.ts', 'revision'], + ['src/storage/snapshot/journal/SnapshotJournalRevisionSql.ts', 'revision'], + ['src/storage/snapshot/journal/SnapshotJournalPage.ts', 'page'], + ['src/storage/snapshot/journal/SnapshotJournalSqliteClock.ts', 'clock'], + ['src/storage/snapshot/journal/SnapshotJournalMysqlClock.ts', 'clock'], + ['src/storage/snapshot/journal/SnapshotJournalSqliteObservers.ts', 'sqlite-observers'], + ['src/storage/snapshot/journal/SnapshotJournalMysqlObservers.ts', 'mysql-observers'], + ['src/storage/snapshot/journal/SnapshotJournalBootstrap.ts', 'bootstrap'], + ['src/storage/snapshot/journal/SnapshotJournalHighWater.ts', 'high-water'], + ['src/storage/snapshot/journal/SnapshotJournalMysqlSource.ts', 'mysql-source'], + ['src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.ts', 'sqlite-generation'], + ['src/storage/snapshot/journal/SnapshotJournalMysqlIntent.ts', 'mysql-intent'], + ['src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.ts', 'mysql-generation'], + ['src/storage/snapshot/journal/SnapshotJournalReceipt.ts', 'receipts'], + ['src/storage/snapshot/journal/SnapshotJournalCaptureFence.ts', 'capture-fence'], + ['src/storage/snapshot/journal/SnapshotJournalConnections.ts', 'connections'], + ['src/storage/snapshot/journal/SnapshotJournalCaptureBackend.ts', 'capture-backend'], + ['src/storage/snapshot/journal/SnapshotJournalCapture.ts', 'capture'], + ['src/storage/snapshot/journal/SnapshotJournalCollection.ts', 'collection'], + ['src/storage/snapshot/journal/SnapshotJournalMaintenance.ts', 'maintenance'], + ['src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.ts', 'maintenance-fence'], + ['src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.ts', 'maintenance-task'] + ]) + } + ], + [ + 'wallet-snapshot-archive', + { + fallback: 'capture', + files: new Map([ + ['src/storage/snapshot/archive/KnexSnapshotArchiveStore.ts', 'store'], + ['src/storage/schema/snapshotArchiveMigration.ts', 'store'], + ['src/storage/snapshot/archive/KnexSnapshotArchiveSource.ts', 'source'], + ['src/storage/snapshot/archive/KnexSnapshotArchiveClosure.ts', 'source'] + ]) + } + ], + [ + 'wallet-snapshot-remote-http', + { + fallback: 'protocol', + files: new Map([ + ['src/storage/remoting/StorageServer.ts', 'server'], + ['src/storage/remoting/StorageClientBase.ts', 'client'] + ]) + } + ], + [ + 'wallet-snapshot-remote-service', + { + fallback: 'persistence', + files: new Map([ + ['src/storage/snapshot/archive/KnexSnapshotArchiveService.ts', 'controller'], + ['src/storage/snapshot/archive/SnapshotArchiveGuard.ts', 'guard'], + ['src/storage/snapshot/archive/SnapshotArchiveGuardRegistry.ts', 'guard'], + ['src/storage/snapshot/archive/SnapshotArchiveGuardBackend.ts', 'backend'] + ]) + } + ], + [ + 'wallet-snapshot-remote-reader', + { + fallback: 'admission', + files: new Map([ + ['src/storage/snapshot/archive/RemoteSnapshotLease.ts', 'lease'], + ['src/storage/snapshot/archive/RemoteSnapshotRows.ts', 'rows'], + ['src/storage/snapshot/archive/RemoteSnapshotPageReader.ts', 'page'], + ['src/storage/snapshot/archive/openRemoteSnapshot.ts', 'page'], + ['src/storage/snapshot/SnapshotCursor.ts', 'page'] + ]) + } ] ]) diff --git a/scripts/mutation-partitions.test.mjs b/scripts/mutation-partitions.test.mjs index 4c48a8b28..a25db5f24 100644 --- a/scripts/mutation-partitions.test.mjs +++ b/scripts/mutation-partitions.test.mjs @@ -1,6 +1,7 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { parseArguments } from './mutation-testing.mjs' +import { parseArguments, REPOSITORY_ROOT } from './mutation-testing.mjs' +import { buildMutationTargets } from '../governance/mutation-testing/targets.mjs' import { partitionMutationTarget, selectedMutationPartition, @@ -18,6 +19,39 @@ const target = { ], runnerOptions: { jest: { config: { testMatch: ['all-original-tests'] } } } } + +test('primary sync partitions preserve complete contracts before dependencies are installed', () => { + const canonical = buildMutationTargets(REPOSITORY_ROOT)['wallet-snapshot-sync'] + const parts = partitionMutationTarget('wallet-snapshot-sync', canonical) + assert.deepEqual( + parts.map(part => part.id), + ['session', 'checkpoint', 'copy', 'storage', 'primary'] + ) + assert.deepEqual(parts.flatMap(part => part.target.mutate).sort(), [...canonical.mutate].sort()) + const owners = new Map() + for (const part of parts) { + assert.deepEqual(part.target, { ...canonical, mutate: part.target.mutate }) + assert.equal(part.target.runnerOptions, canonical.runnerOptions) + for (const specification of part.target.mutate) { + const file = specification.replace(/:\d+(?:-\d+)?$/, '') + assert.ok(!owners.has(file) || owners.get(file) === part.id) + owners.set(file, part.id) + } + } + assert.equal(owners.get('src/storage/WalletStorageManager.ts'), 'primary') + assert.equal(owners.get('src/storage/StorageKnex.ts'), 'storage') + assert.equal(owners.get('src/storage/snapshot/SnapshotSync.ts'), 'session') + const helper = 'src/storage/snapshot/FuturePrimaryHelper.ts' + const future = partitionMutationTarget('wallet-snapshot-sync', { + ...canonical, + mutate: [...canonical.mutate, helper] + }) + assert.deepEqual(future[0].target.mutate, [...parts[0].target.mutate, helper]) + assert.equal(selectedMutationPartition('wallet-snapshot-sync', canonical), canonical) + const targets = buildMutationTargets(REPOSITORY_ROOT) + assert.ok(mutationExecutionMatrix(Object.keys(targets), targets).include.length <= 256) +}) + test('execution partitions preserve complete original specifications and identical full test configuration', () => { const parts = partitionMutationTarget('sdk-auth-http', target) assert.deepEqual( @@ -142,3 +176,199 @@ test('retained partitions preserve complete lifecycle/reader/storage unions and ) assert.throws(() => partitionedMutationTargets(['sdk-auth-http', 'sdk-auth-http'], targets)) }) + +test('service execution preserves the complete canonical union and all configuration in every part', () => { + const canonical = buildMutationTargets(REPOSITORY_ROOT)['wallet-snapshot-remote-service'] + const parts = partitionMutationTarget('wallet-snapshot-remote-service', canonical) + assert.deepEqual( + parts.map(part => part.id), + ['persistence', 'controller', 'guard', 'backend'] + ) + assert.deepEqual(parts.flatMap(part => part.target.mutate).sort(), [...canonical.mutate].sort()) + assert.equal(new Set(parts.flatMap(part => part.target.mutate)).size, canonical.mutate.length) + for (const part of parts) { + assert.deepEqual(part.target, { ...canonical, mutate: part.target.mutate }) + assert.equal(part.target.runnerOptions, canonical.runnerOptions) + } + assert.deepEqual(parts[1].target.mutate, [ + 'src/storage/snapshot/archive/KnexSnapshotArchiveService.ts' + ]) + assert.deepEqual(parts[2].target.mutate, [ + 'src/storage/snapshot/archive/SnapshotArchiveGuard.ts', + 'src/storage/snapshot/archive/SnapshotArchiveGuardRegistry.ts' + ]) + assert.deepEqual(parts[3].target.mutate, [ + 'src/storage/snapshot/archive/SnapshotArchiveGuardBackend.ts' + ]) + const future = { + ...canonical, + mutate: [...canonical.mutate, 'src/storage/snapshot/archive/FutureHelper.ts'] + } + const expanded = partitionMutationTarget('wallet-snapshot-remote-service', future) + assert.equal(expanded[0].id, 'persistence') + assert.ok(expanded[0].target.mutate.includes('src/storage/snapshot/archive/FutureHelper.ts')) + assert.equal(selectedMutationPartition('wallet-snapshot-remote-service', canonical), canonical) + assert.throws(() => + selectedMutationPartition('wallet-snapshot-remote-service', canonical, 'missing') + ) + for (const specification of ['src/**/*.ts', '!src/helper.ts', '../outside.ts', '/outside.ts']) { + assert.throws(() => + partitionMutationTarget('wallet-snapshot-remote-service', { mutate: [specification] }) + ) + } + const targets = { before: target, 'wallet-snapshot-remote-service': canonical, after: target } + assert.deepEqual(mutationExecutionMatrix(Object.keys(targets), targets).include, [ + { target: 'before', partition: 'whole' }, + ...['persistence', 'controller', 'guard', 'backend'].map(partition => ({ + target: 'wallet-snapshot-remote-service', + partition + })), + { target: 'after', partition: 'whole' } + ]) +}) + +test('HTTP execution preserves every canonical range, full configuration and future fallback', () => { + const canonical = buildMutationTargets(REPOSITORY_ROOT)['wallet-snapshot-remote-http'] + const parts = partitionMutationTarget('wallet-snapshot-remote-http', canonical) + assert.deepEqual( + parts.map(part => part.id), + ['protocol', 'client', 'server'] + ) + const original = [...canonical.mutate].sort() + const actual = parts.flatMap(part => part.target.mutate) + assert.deepEqual([...actual].sort(), original) + assert.equal(new Set(actual).size, original.length) + for (const part of parts) { + assert.deepEqual(part.target, { ...canonical, mutate: part.target.mutate }) + assert.equal(part.target.runnerOptions, canonical.runnerOptions) + } + assert.deepEqual(parts[0].target.mutate, [ + 'src/storage/snapshot/archive/SnapshotArchiveProtocol.ts', + 'src/storage/snapshot/archive/KnexSnapshotArchiveRpc.ts', + 'src/storage/snapshot/archive/SnapshotArchiveTransport.ts' + ]) + for (const [index, file] of [ + [1, 'StorageClientBase.ts'], + [2, 'StorageServer.ts'] + ]) { + assert.deepEqual( + parts[index].target.mutate, + canonical.mutate.filter(specification => + specification.startsWith(`src/storage/remoting/${file}:`) + ) + ) + assert.ok(parts[index].target.mutate.length > 1) + } + const helper = 'src/storage/snapshot/archive/FutureHttpHelper.ts' + const expanded = partitionMutationTarget('wallet-snapshot-remote-http', { + ...canonical, + mutate: [...canonical.mutate, helper] + }) + assert.equal(expanded[0].id, 'protocol') + assert.deepEqual(expanded[0].target.mutate, [...parts[0].target.mutate, helper]) + assert.equal(selectedMutationPartition('wallet-snapshot-remote-http', canonical), canonical) + assert.throws(() => + selectedMutationPartition('wallet-snapshot-remote-http', canonical, 'missing') + ) + for (const specification of ['src/**/*.ts', '!src/helper.ts', '../outside.ts', '/outside.ts']) { + assert.throws(() => + partitionMutationTarget('wallet-snapshot-remote-http', { mutate: [specification] }) + ) + } + const targets = { before: target, 'wallet-snapshot-remote-http': canonical, after: target } + assert.deepEqual(mutationExecutionMatrix(Object.keys(targets), targets).include, [ + { target: 'before', partition: 'whole' }, + ...['protocol', 'client', 'server'].map(partition => ({ + target: 'wallet-snapshot-remote-http', + partition + })), + { target: 'after', partition: 'whole' } + ]) +}) + +for (const [id, expected, fallback] of [ + [ + 'wallet-retained-snapshot', + [ + 'lifecycle', + 'reader', + 'profile-index', + 'relation-index', + 'certificate-index', + 'global-index', + 'global-mysql', + 'global-sqlite', + 'global-bootstrap', + 'global-triggers', + 'sqlite-identity', + 'sqlite-membership', + 'sqlite-generation', + 'sqlite-bootstrap', + 'sqlite-retirement', + 'storage' + ], + 'lifecycle' + ], + [ + 'wallet-snapshot-journal', + [ + 'revision', + 'page', + 'clock', + 'sqlite-observers', + 'mysql-observers', + 'bootstrap', + 'high-water', + 'mysql-source', + 'sqlite-generation', + 'mysql-intent', + 'mysql-generation', + 'receipts', + 'capture-fence', + 'connections', + 'capture-backend', + 'capture', + 'collection', + 'maintenance', + 'maintenance-fence', + 'maintenance-task' + ], + 'revision' + ], + ['wallet-snapshot-archive', ['store', 'capture', 'source'], 'capture'], + ['wallet-snapshot-remote-reader', ['admission', 'lease', 'rows', 'page'], 'admission'] +]) { + test(`${id} preserves canonical whole-file ownership, configuration and future source coverage`, () => { + const canonical = buildMutationTargets(REPOSITORY_ROOT)[id] + const parts = partitionMutationTarget(id, canonical) + assert.deepEqual( + parts.map(part => part.id), + expected + ) + assert.deepEqual(parts.flatMap(part => part.target.mutate).sort(), [...canonical.mutate].sort()) + const owners = new Map() + for (const part of parts) { + const { mutate } = part.target + assert.deepEqual(part.target, { ...canonical, mutate }) + assert.equal(part.target.runnerOptions, canonical.runnerOptions) + for (const specification of mutate) { + const file = specification.replace(/:\d+(?:-\d+)?$/, '') + assert.ok(!owners.has(file) || owners.get(file) === part.id) + owners.set(file, part.id) + } + } + const helper = 'src/storage/snapshot/FutureHelper.ts' + const extended = { ...canonical, mutate: [...canonical.mutate, helper, `${helper}:1-20`] } + const expanded = partitionMutationTarget(id, extended) + assert.deepEqual( + expanded.flatMap(part => part.target.mutate).sort(), + [...extended.mutate].sort() + ) + assert.deepEqual(expanded.find(part => part.id === fallback).target.mutate.slice(-2), [ + helper, + `${helper}:1-20` + ]) + assert.equal(selectedMutationPartition(id, canonical), canonical) + assert.throws(() => selectedMutationPartition(id, canonical, 'missing')) + }) +} diff --git a/scripts/mutation-testing.mjs b/scripts/mutation-testing.mjs index 8653575a9..8e45ff653 100644 --- a/scripts/mutation-testing.mjs +++ b/scripts/mutation-testing.mjs @@ -73,6 +73,9 @@ function targetInputPatterns(target) { for (const mutate of target.mutate ?? []) { patterns.push(`${packageDirectory}/${mutate.replace(/:\d+(?:-\d+)?$/, '')}`) } + for (const input of target.additionalInputs ?? []) { + patterns.push(`${packageDirectory}/${normalized(input)}`) + } if (typeof target.propertyTest === 'string') patterns.push(normalized(target.propertyTest)) const jest = target.runnerOptions?.jest const vitest = target.runnerOptions?.vitest diff --git a/scripts/mutation-testing.test.mjs b/scripts/mutation-testing.test.mjs index 0b460053b..6b29999dd 100644 --- a/scripts/mutation-testing.test.mjs +++ b/scripts/mutation-testing.test.mjs @@ -1,10 +1,13 @@ +import { readFileSync } from 'node:fs' import assert from 'node:assert/strict' import test from 'node:test' +import { buildMutationTargets } from '../governance/mutation-testing/targets.mjs' import { calculateMutationMetrics, evaluateMutationReport, parseArguments, + REPOSITORY_ROOT, selectAffectedMutationTargets, strykerEnvironment, targetsForUnresolvedMutationRange @@ -163,6 +166,219 @@ test('mutation report evaluation ratchets score, coverage, and invalid outcomes' ) }) +test('additional package-relative fixture inputs select their target without replacing existing inputs', () => { + const target = { + packageDirectory: 'packages/one', + propertyTest: 'packages/one/test/value.property.test.ts', + mutate: ['src/value.ts:10-20'], + additionalInputs: ['./test/fixtures/source.ts'], + runnerOptions: { + jest: { + configFile: 'jest.config.cjs', + config: { testMatch: ['/test/value*.test.ts'] } + } + } + } + const precise = { + one: target, + two: { + ...target, + packageDirectory: 'packages/two', + propertyTest: 'packages/two/test/value.property.test.ts' + } + } + for (const input of [ + 'src/value.ts', + 'test/value.property.test.ts', + 'jest.config.cjs', + 'test/value.test.ts', + 'test/fixtures/source.ts' + ]) { + assert.deepEqual(selectAffectedMutationTargets(precise, [`packages/one/${input}`]), ['one']) + assert.deepEqual(selectAffectedMutationTargets(precise, [`packages/two/${input}`]), ['two']) + } + for (const input of [ + 'test/fixtures/other.ts', + 'test/fixtures/source.ts.extra', + 'src/other.ts', + 'packages/one/test/fixtures/source.ts' + ]) { + assert.deepEqual(selectAffectedMutationTargets(precise, [`packages/one/${input}`]), []) + } + assert.deepEqual(selectAffectedMutationTargets(precise, ['test/fixtures/source.ts']), []) + assert.deepEqual( + selectAffectedMutationTargets(precise, ['packages/one/test/fixtures/source.ts'], { + changedTargetIds: ['two'] + }), + ['one', 'two'] + ) + + const canonical = buildMutationTargets(REPOSITORY_ROOT) + assert.equal(Object.keys(canonical).length, 48) + assert.deepEqual(canonical['wallet-retained-snapshot'].additionalInputs, [ + 'test/utils/snapshotRelationFixtures.ts', + 'test/utils/snapshotCertificateFixtures.ts', + 'test/utils/snapshotGlobalFixtures.ts', + 'test/utils/snapshotHistoricalMigrations.ts', + 'test/utils/snapshotSqliteFixtures.ts', + 'test/utils/snapshotSqliteIdentityFixture.ts', + 'test/utils/snapshotSqliteMaintenanceFixture.ts', + 'test/storage/snapshotHistoricalMigrations.cjs', + 'test/storage/snapshotSqliteGenerationCrash.cjs' + ]) + assert.deepEqual( + selectAffectedMutationTargets(canonical, [ + 'packages/wallet/wallet-toolbox/test/utils/snapshotRelationFixtures.ts' + ]), + ['wallet-retained-snapshot'] + ) + for (const input of [ + 'src/storage/schema/snapshotCertificateIndexMigration.ts', + 'src/storage/schema/snapshotGlobalIndexMigration.ts', + 'src/storage/schema/snapshotGlobalIndexTriggers.ts', + 'test/utils/snapshotCertificateFixtures.ts', + 'test/utils/snapshotGlobalFixtures.ts', + 'src/storage/schema/snapshotRelationIndexMigration.ts', + 'src/storage/schema/snapshotProfileIndexMigration.ts', + 'src/storage/snapshot/RetainedReadSnapshot.property.test.ts' + ]) { + assert.ok( + selectAffectedMutationTargets(canonical, [ + `packages/wallet/wallet-toolbox/${input}` + ]).includes('wallet-retained-snapshot') + ) + } +}) + +test('retained snapshot mutation execution recycles workers while other wallet defaults remain intact', () => { + const targets = buildMutationTargets(REPOSITORY_ROOT) + assert.equal(targets['wallet-retained-snapshot'].runnerOptions.maxTestRunnerReuse, 8) + assert.equal(targets['wallet-snapshot-archive'].runnerOptions.maxTestRunnerReuse, undefined) + assert.equal(targets['wallet-snapshot-remote-http'].runnerOptions.maxTestRunnerReuse, undefined) +}) + +test('journal mutation registration retains its complete source, canonical tests and fixture ownership', () => { + const target = buildMutationTargets(REPOSITORY_ROOT)['wallet-snapshot-journal'] + assert.deepEqual(target.mutate, [ + 'src/storage/snapshot/journal/SnapshotJournalRevision.ts', + 'src/storage/snapshot/journal/SnapshotJournalRevisionSql.ts', + 'src/storage/snapshot/journal/SnapshotJournalPage.ts', + 'src/storage/snapshot/journal/SnapshotJournalSqliteClock.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlClock.ts', + 'src/storage/snapshot/journal/SnapshotJournalSqliteObservers.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlObservers.ts', + 'src/storage/snapshot/journal/SnapshotJournalBootstrap.ts', + 'src/storage/snapshot/journal/SnapshotJournalHighWater.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlSource.ts', + 'src/storage/snapshot/journal/SnapshotJournalSqliteGeneration.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlIntent.ts', + 'src/storage/snapshot/journal/SnapshotJournalMysqlGeneration.ts', + 'src/storage/snapshot/journal/SnapshotJournalReceipt.ts', + 'src/storage/snapshot/journal/SnapshotJournalCaptureFence.ts', + 'src/storage/snapshot/journal/SnapshotJournalConnections.ts', + 'src/storage/snapshot/journal/SnapshotJournalCaptureBackend.ts', + 'src/storage/snapshot/journal/SnapshotJournalCapture.ts', + 'src/storage/snapshot/journal/SnapshotJournalCollection.ts', + 'src/storage/snapshot/journal/SnapshotJournalMaintenance.ts', + 'src/storage/snapshot/journal/SnapshotJournalMaintenanceFence.ts', + 'src/storage/snapshot/journal/SnapshotJournalMaintenanceTask.ts' + ]) + assert.deepEqual(target.additionalInputs, [ + 'test/fixtures/snapshotJournal/mysql-generation-ddl-fixture.json', + 'test/fixtures/snapshotJournal/mysql-generation-metadata-fixture.json', + 'test/fixtures/snapshotJournal/mysql-generation-state-fixture.json', + 'test/fixtures/snapshotJournal/mysql-intent-metadata-fixture.json', + 'test/fixtures/snapshotJournal/mysql-source-metadata-fixture.json', + 'test/utils/snapshotArchiveFixtures.ts', + 'test/utils/snapshotSqliteFixtures.ts', + 'test/utils/snapshotHistoricalMigrations.ts', + 'test/storage/snapshotJournalNativeFixture.cjs', + 'test/storage/snapshotJournalMysqlConnection.cjs', + 'test/storage/snapshotJournalMysql.cjs', + 'test/storage/snapshotJournalMysqlServerCrash.cjs', + 'test/storage/snapshotJournalReceiptMysql.cjs', + 'test/storage/snapshotJournalCaptureMysql.cjs', + 'test/storage/snapshotJournalCaptureMysqlChild.cjs', + 'test/storage/snapshotJournalCaptureProcessLoss.cjs', + 'test/storage/snapshotJournalCaptureSqlite.cjs', + 'test/storage/snapshotJournalRetentionChild.cjs', + 'test/storage/snapshotJournalRetentionCuts.cjs', + 'test/storage/snapshotJournalRetentionMysql.cjs', + 'test/storage/snapshotJournalRetentionMysqlRc.cjs', + 'test/storage/snapshotJournalRetentionMysqlRr.cjs', + 'test/storage/snapshotJournalRetentionProcessLoss.cjs', + 'test/storage/snapshotJournalRetentionSqlite.cjs', + 'test/storage/snapshotJournalMaintenanceFixture.cjs', + 'test/storage/snapshotJournalMaintenanceCuts.cjs', + 'test/storage/snapshotJournalMaintenanceChild.cjs', + 'test/storage/snapshotJournalMaintenanceProcessLoss.cjs', + 'test/storage/snapshotJournalMaintenanceWal.cjs', + 'test/storage/snapshotJournalMaintenanceMysqlRc.cjs', + 'test/storage/snapshotJournalMaintenanceMysqlRr.cjs', + 'test/storage/snapshotJournalSqliteCrash.cjs', + 'test/storage/runSnapshotJournalMysql.cjs', + 'test/storage/snapshotArchiveDocker.cjs' + ]) + assert.equal(target.runnerOptions.maxTestRunnerReuse, 8) + assert.deepEqual(target.runnerOptions.jest.config.testMatch, [ + '/src/storage/snapshot/journal/*.test.ts' + ]) + for (const input of target.additionalInputs) + assert.ok( + selectAffectedMutationTargets(buildMutationTargets(REPOSITORY_ROOT), [ + 'packages/wallet/wallet-toolbox/' + input + ]).includes('wallet-snapshot-journal') + ) +}) + +test('every inherited snapshot-sync target keeps the complete journal tests for owned provider ranges', () => { + const targets = buildMutationTargets(REPOSITORY_ROOT) + for (const name of [ + 'wallet-snapshot-sync', + 'wallet-snapshot-sync-destination', + 'wallet-snapshot-sync-rows' + ]) { + assert.ok( + targets[name].runnerOptions.jest.config.testMatch.includes( + '/src/storage/snapshot/journal/*.test.ts' + ) + ) + assert.ok( + targets[name].runnerOptions.jest.config.testMatch.includes( + '/src/storage/snapshot/SnapshotSync*.test.ts' + ) + ) + } +}) + +test('snapshot sync owns every inherited manager region and complete primary selection', () => { + const target = buildMutationTargets(REPOSITORY_ROOT)['wallet-snapshot-sync'] + const file = 'src/storage/WalletStorageManager.ts' + const lines = readFileSync( + `${REPOSITORY_ROOT}/packages/wallet/wallet-toolbox/${file}`, + 'utf8' + ).split('\n') + const expected = [ + ['private async runSnapshotCopy(', 'async syncFromReader('], + ['async syncFromReader(', ' let inserts = 0'], + ['async syncFromReaderResumable(', ' const generation ='], + ['async syncToWriterResumable(', 'async syncToWriter('], + ['async syncToWriter(', ' let inserts = 0'], + ['async updateBackups(', 'async setActive('], + ['async setActive(', 'getStoreEndpointURL('], + ['private async withAccess(', 'runAsWriter('] + ].map(([startMarker, endMarker]) => { + const start = lines.findIndex(line => line.includes(startMarker)) + const end = lines.findIndex((line, index) => index > start && line.includes(endMarker)) + assert.ok(start >= 0 && end > start) + return `${file}:${start + 1}-${end}` + }) + assert.deepEqual( + target.mutate.filter(specification => specification.startsWith(`${file}:`)), + expected + ) +}) + test('Stryker runs cannot append per-mutant test reports to the job summary', () => { const environment = strykerEnvironment( { GITHUB_ACTIONS: 'true', GITHUB_STEP_SUMMARY: '/tmp/summary', PATH: '/bin' }, diff --git a/scripts/patch-coverage.mjs b/scripts/patch-coverage.mjs index 59d693110..78c2208b0 100644 --- a/scripts/patch-coverage.mjs +++ b/scripts/patch-coverage.mjs @@ -197,30 +197,24 @@ export function omitStaticMarkdownModules(changed, readSource) { } } -function omitTypeOnlyChanges(changed, base) { - const mergeBase = execFileSync('/usr/bin/git', ['merge-base', base, 'HEAD'], { - cwd: REPOSITORY_ROOT, - encoding: 'utf8' - }).trim() - for (const file of changed.keys()) { - if (!/\.[cm]?ts$/.test(file)) continue - let before - try { - before = execFileSync('/usr/bin/git', ['show', `${mergeBase}:${file}`], { - cwd: REPOSITORY_ROOT, - encoding: 'utf8', - stdio: ['ignore', 'pipe', 'ignore'], - maxBuffer: 20 * 1024 * 1024 - }) - } catch { - // New or unreadable files remain governed. - continue - } - const after = execFileSync('/usr/bin/git', ['show', `HEAD:${file}`], { - cwd: REPOSITORY_ROOT, +export function omitTypeOnlyChanges(changed, base, repositoryRoot = REPOSITORY_ROOT) { + const git = arguments_ => + execFileSync('/usr/bin/git', arguments_, { + cwd: repositoryRoot, encoding: 'utf8', + stdio: ['ignore', 'pipe', 'ignore'], maxBuffer: 20 * 1024 * 1024 }) + const mergeBase = git(['merge-base', base, 'HEAD']).trim() + // Only a successfully read base tree can prove that a path is new. A failed + // blob read is not evidence of an empty file: Git/read errors fail the gate. + const baseFiles = new Set( + git(['ls-tree', '-r', '--name-only', '-z', mergeBase, '--', 'packages']).split('\0') + ) + for (const file of changed.keys()) { + if (!/\.[cm]?ts$/.test(file)) continue + const before = baseFiles.has(file) ? git(['show', `${mergeBase}:${file}`]) : '' + const after = git(['show', `HEAD:${file}`]) if (!hasRuntimeChange(before, after)) changed.delete(file) } } diff --git a/scripts/patch-coverage.test.mjs b/scripts/patch-coverage.test.mjs index 0e8f060db..ce13a5f0d 100644 --- a/scripts/patch-coverage.test.mjs +++ b/scripts/patch-coverage.test.mjs @@ -1,6 +1,9 @@ import assert from 'node:assert/strict' import test from 'node:test' -import { readFileSync } from 'node:fs' +import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' +import { execFileSync } from 'node:child_process' import { changedLinesFromDiff, @@ -8,6 +11,7 @@ import { hasRuntimeChange, isStaticMarkdownModule, omitStaticMarkdownModules, + omitTypeOnlyChanges, mergeLcov, runtimeComparisonAvailable } from './patch-coverage.mjs' @@ -281,3 +285,67 @@ test('the coverage aggregation job installs its locked compiler before classifyi const manifest = JSON.parse(readFileSync(new URL('../package.json', import.meta.url), 'utf8')) assert.equal(typeof manifest.devDependencies.esbuild, 'string') }) + +test('new erased declarations need no LCOV while runtime, unsupported syntax and unreadable Git sources stay governed', t => { + const repository = mkdtempSync(join(tmpdir(), 'patch-coverage-source-')) + t.after(() => rmSync(repository, { recursive: true, force: true })) + const git = (...args) => + execFileSync('/usr/bin/git', args, { + cwd: repository, + encoding: 'utf8', + stdio: ['ignore', 'pipe', 'pipe'] + }).trim() + const write = (name, source) => writeFileSync(join(repository, 'packages', name), source) + git('init', '--initial-branch=main') + mkdirSync(join(repository, 'packages')) + write('existing.ts', 'export interface Options { first: string }') + git('add', 'packages') + const commit = () => + git( + '-c', + 'user.name=Coverage fixture', + '-c', + 'user.email=fixture@example.invalid', + 'commit', + '-m', + 'Synthetic source' + ) + commit() + const base = git('rev-parse', 'HEAD') + write('existing.ts', 'export interface Options { first: string; second?: number }') + write( + 'new-types.ts', + "import type * as values from './values'; export type Value = values.Value; export interface Options { value: Value }" + ) + write('new-runtime.ts', 'export const value = 1') + write('new-enum.ts', 'export enum State { Ready }') + write('new-effect.ts', 'import "./start.js"; export interface Options {}') + write('new-invalid.ts', 'invalid TypeScript {') + git('add', 'packages') + commit() + const names = [ + 'existing.ts', + 'new-types.ts', + 'new-runtime.ts', + 'new-enum.ts', + 'new-effect.ts', + 'new-invalid.ts' + ] + const changed = new Map(names.map(name => ['packages/' + name, new Set([1])])) + omitTypeOnlyChanges(changed, base, repository) + const expected = runtimeComparisonAvailable() ? names.slice(2) : names + assert.deepEqual( + [...changed.keys()], + expected.map(name => 'packages/' + name) + ) + assert.throws(() => + omitTypeOnlyChanges(new Map([['packages/missing.ts', new Set([1])]]), base, repository) + ) + assert.throws(() => + omitTypeOnlyChanges( + new Map([['packages/new-types.ts', new Set([1])]]), + 'missing-base', + repository + ) + ) +}) diff --git a/scripts/test-governance.test.mjs b/scripts/test-governance.test.mjs index 808682aeb..01e116ada 100644 --- a/scripts/test-governance.test.mjs +++ b/scripts/test-governance.test.mjs @@ -32,11 +32,11 @@ test('current required, manual, live, resource, and conformance tests are govern assert.deepEqual(result.errors, []) assert.equal(result.summary.requiredDirectSkips, 2) - assert.equal(result.summary.propertySuites, 37) + assert.equal(result.summary.propertySuites, 48) assert.equal(result.summary.propertyPackages, 31) assert.equal(result.summary.propertyExcludedPackages, 5) assert.equal(result.summary.propertyClassifiedPackages, 36) - assert.equal(result.summary.mutationTargets, 37) + assert.equal(result.summary.mutationTargets, 48) assert.equal(result.summary.manualAndLiveFiles, 32) assert.equal(result.summary.walletManualSuites, 30) assert.equal(result.summary.conformanceSkipFiles, 19) diff --git a/sonar-project.properties b/sonar-project.properties index e51a435da..f7dfe98ff 100644 --- a/sonar-project.properties +++ b/sonar-project.properties @@ -73,7 +73,11 @@ infra/wab/src/telemetry.ts,\ infra/uhrp-server-basic/src/telemetry.ts,\ infra/uhrp-server-cloud-bucket/src/telemetry.ts,\ infra/wallet-infra/src/telemetry.ts,\ -infra/message-box-server/src/telemetry.ts +infra/message-box-server/src/telemetry.ts,\ +infra/uhrp-server-cloud-bucket/dev/bin/nodemon.js,\ +infra/uhrp-server-cloud-bucket/dev/check-watch.cjs,\ +infra/wab/dev/bin/nodemon.js,\ +infra/wab/dev/check-watch.cjs # Keep CI and Automatic Analysis aligned on the same narrowly registered # compatibility exceptions. sonar.issue.ignore.multicriteria=werrProtocolNames,curveSingletonAlias,curveSingletonReturn,scriptOpcodeDispatch diff --git a/specs/wallet/sync-portability-program.md b/specs/wallet/sync-portability-program.md new file mode 100644 index 000000000..3492c35d4 --- /dev/null +++ b/specs/wallet/sync-portability-program.md @@ -0,0 +1,347 @@ +# Wallet sync and portability implementation program + +Issue [#544](https://github.com/bsv-blockchain/ts-stack/issues/544) is the acceptance +contract for this program on [#569](https://github.com/bsv-blockchain/ts-stack/pull/569). +The PR remains open, unmerged and draft while implementation or qualification is +incomplete. A green intermediate source revision does not complete this program. + +The numeric relation checkpoint extends auxiliary indexed selection to +`tx_labels_map` and `output_tags_map`. It preserves composite order and both parent +ownership bases, retains inconsistent mappings for closure refusal, and resumes +256-row bootstrap batches after interrupted DDL or transaction commits. Retained +ordinary/archive readers choose the complete migration inside their pinned view. +Native fixtures cover seven migrator process-loss boundaries, independent writer +locks under both MySQL isolation levels and late-page range/read-count evidence +both before and after native optimizer statistics refresh. +Repository and exact-head qualification must still complete for this checkpoint. +The subsequent certificate-field checkpoint preserves text collation and exact +field names while indexing both direct and parent ownership. Its resumable +migration and retained-view adoption include ordinary and archive paths; native +process-loss, concurrent-writer and bounded-page qualification remain mandatory. +The global proof/request checkpoint adds reference edges, exact per-profile +counts and presence guards, including independent current-read writers and +resumable bounded bootstrap. Ordinary/archive selection uses complete state in +the retained view. All thirteen standard tables now have auxiliary indexed +selection paths; complete native, mutation and exact-head qualification remain +required. Commit ordering, tombstones, nonblocking IndexedDB and the remaining +program below remain open; this does not complete S2. + +## Baseline and immediate defect + +At `ec12ee79deb69d2019b3e50ee70d975752293d0b`, `syncFromReaderResumable` yields +between atomic local destination commits. `syncToWriter`, ordinary +`syncFromReader`, `updateBackups` and primary reconciliation still hold exclusive +manager ownership across entire copy loops. Near-realtime backup therefore still +blocks foreground wallet operations. Existing benchmarks cover pull scheduling, +not this push/backup acceptance case. + +Primary selection now checks apparent no-op requests inside the existing +exclusive ownership boundary. Queued B then A requests therefore finish with A +selected, including when the earlier switch fails. The progress formatter runs +before generation advancement so a formatting failure before the transition +does not invalidate an in-flight copy. Regression cases and generated queued +histories cover this correction. Partial propagation failure invalidates cached +primary authorization and managed user rows before ownership is released. Queued +requests reload persisted selections after acquiring ownership; a reload failure +preserves its cause and releases the queue. Conflicting persisted selections +refuse active authorization until explicitly reconciled. Primary reconciliation +still holds ownership across its copy loop; these corrections do not complete S1. + +Removing those locks alone is insufficient. A source can change between live +offset pages, and an old source reply can overlap primary replacement. Source +isolation, checkpoint ownership and generation fencing must accompany yielding. +Existing BRC-38 capture also reads tables independently and materializes the +complete archive; a local replica export cannot stand in for original remote +source metadata and sync-state history. + +## Required checkpoints + +Each row remains open until its implementation **and** evidence are complete. +The sequence permits intermediate commits, not permanent scope deferrals. + +| ID | Required result | Code and evidence boundary | Baseline | +| --- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------- | +| S1 | Both directions, ordinary backup and primary reconciliation use bounded resumable work; foreground reads/writes proceed outside atomic commits; fair background admission; explicit cancellation and stale-session rejection | `WalletStorageManager`, `storage/sync`; delayed peer, queued cancellation, lost acknowledgement, concurrent sessions, replacement and restart tests; push and pull traces | Pull subset only | +| S2 | Coherent provider read views, stable indexed keysets and immutable high-water positions preserve closure, equal timestamps and tombstones under monitor/device writes | `StorageProvider`, `StorageKnex`, `StorageIdb`, schemas and migrations; independent writers, process termination, query plans and bounded page tests | Missing coherent view/keysets | +| S3 | Negotiated authenticated single-profile remote snapshot/export handles bind identity, network, source schema/version and immutable content; quotas, expiry, bounded cursors and cleanup | `StorageClientBase`, `StorageClient`, `StorageServer`, shared contracts; real authenticated HTTP, old/new combinations and expiry/restart evidence | Missing | +| S4 | Bounded metadata/blob transport with integrity-checked content reuse, explicit row/byte/in-flight limits, adaptive page control, no repeated full counts/maps, optional measured read-ahead with ordered commits | Sync transfer, provider queries and packed adapters; large individual row, high fixed latency, table-transition and backpressure tests | Binary transfer/controller subset only | +| P1 | Canonical streaming BRC-38 contains all standard tables and relations plus original source provenance/sync state from one snapshot; optional legacy object fields normalize without mutation or lost array values | `storage/portable`, shared vectors and independent reader/writer; full closure, nullable history, binary/order and concurrent capture tests | Materialized non-snapshot capture | +| P2 | Bounded BRC-39 encryption/decryption and file processing retain the exact envelope, NFC password handling, canonical Argon2id strength and complete authentication before activation | Portable codec, worker/native backends and bounded transferable/IPC adapters; independent bytes, wrong password, corrupt/truncated/oversized inputs, progress/cancel and memory evidence | Materialized file processing | +| P3 | Import preview, isolated durable staging, semantic validation, ID remapping, repeat import and a durable commit/recovery journal; preserve provenance/primary history without automatic activation | Portable import, storage migrations/journal; kill/restart at every durable boundary, empty/occupied targets, repeat merge and disk-pressure tests | Legacy restore/merge subset only | +| A1 | One wallet identity/network/profile throughout every export/import/sync; supported versioned packed IDB/native adapters and downstream migration removing coarse queues/private schema copies | Public core/client/mobile artifacts and host example; independent profiles, native browser, supported native databases and mobile lifecycle | Partial existing platform artifacts | +| V1 | Reproducible per-backend and combined performance/fault/conformance evidence with foreground p50/p95/p99, lock/queue time, cancellation, wall/wire/CPU/query/memory costs and quiescent unchanged copies | SQLite, IndexedDB, authenticated remote; small/large wallets and rows, cold/incremental/no-change/export/import, latency/disconnect cohorts | Pull-only benchmark baseline | +| V2 | Complete affected packages, packed consumers, docs, conformance, property/mutation, analyzer and exact-head hosted repository gates | Root governance and CI; no relaxed thresholds, hidden findings or substituted platform claims | Earlier head qualified only | +| R1 | Released packages, provider rollout, consumer migration and explicitly authorized funded restore/spend plus required physical platform drills | Protected release/deployment workflows and host/operator acceptance records | Separate authorization and execution required | + +## Compatibility and data boundaries + +- Additive capabilities precede their use. Unsupported providers retain the safe + serialized path and explicitly refuse unsupported complete source exports. + Old/new combinations must not silently advertise stronger guarantees. +- Preserve released APIs, BRC-38/39/40 encodings, error identities and inclusive + timestamp behavior. New cursor/schema contracts are versioned and tested with + resumable upgrades and documented downgrade constraints. +- BRC-38 v1 deliberately excludes pending action batches, auxiliary runtime + tables, application-specific tables and storage-global logs. Do not settle + pending batches or invent an archive extension to make export work. +- Preserve historical managed-change policy in archive bytes. Operational + migration is separate from faithful export/import provenance. +- Key recovery and wallet-data recovery remain distinct. All automated fixtures + use synthetic records; production keys, archives, payloads and identities do + not enter telemetry, public coordination or source evidence. +- Keep all memory, temporary disk, retention, queue and KDF limits explicit. + Cancellation before commit discards staged work; cancellation during a commit + reports the durable outcome before stopping. No unverified plaintext becomes + active wallet data. + +## Completion evidence + +The first implementation checkpoint adds local provider read views and threads +their token through every BRC-38 capture query, including uncached source +settings. Built-in providers use that view automatically. `requireSnapshot` +requests the guarantee explicitly and refuses unsupported providers before table +reads; existing custom-provider calls keep their caller-quiesced compatibility +path. Tests cover an independent SQLite WAL writer, queued IndexedDB writes, +failure cleanup, profile separation and stale settings caches. Optional legacy +JSON fields normalize without changing source history; required values and array +entries cannot be silently dropped. Inconsistent owned label/tag mappings reject +capture instead of being filtered out of the archive. + +The next checkpoint adds a bounded local SQL read-view lifetime. Opening pins a +SQLite/MySQL view before returning, retains it across idle reads, rejects nested +or concurrent reads, and keeps provider capacity occupied until physical cleanup +after close, expiry, cancellation or read failure. Tests cover exact lifetime +boundaries, delayed acquisition/initialization/read/cleanup, independent writers, +profile-filtered reads and randomized operation schedules. The local methods are +absent from the RPC allowlist. Each view occupies one pool connection; query +cancellation/deadlines remain a driver concern. Retention is explicitly +unsupported on IndexedDB. No persisted schema or index transition is introduced. + +The subsequent local SQL page contract binds one profile and source metadata to +the retained view, returns all thirteen standard tables through keyset cursors, +and preflights payload sizes before fetching rows. Binary fields remain packed. +Retries within a view repeat pages; a new view rejects its predecessor's cursor. +Existing unique keys avoid changing legacy OFFSET traversal. Composite storage +order is explicit and differs from canonical archive order. This provides +bounded local pages, with explicit oversized-row refusal pending streaming. + +The durable local integration checkpoint connects supported ordinary push, pull +and backup calls to those pages. A dedicated SQLite-WAL/static-MySQL reader leaves +foreground pool capacity available. The destination commits entity rows, +normalized bounded ID mappings and its cursor atomically. An additive auxiliary +migration and database-owned primary epoch fence stale pages, including independent +away-and-back changes. Same-view lost acknowledgements resume the destination +checkpoint; replacement source views restart traversal with retained mappings. +Tests terminate the destination process before a row, before checkpoint update, +before transaction commit and after commit/before acknowledgement, then recover +and finish without duplicate rows or partial checkpoints. + +The subsequent internal SQL staging component adds shared immutable-page storage +for remote snapshots. It binds profile/source metadata, reserves bounded logical +capacity, commits pages and retry receipts atomically, and retains capacity until +resumable cleanup finishes. Independent-connection SQLite/MySQL and actual SQLite +process-termination fixtures cover its persistence boundaries. The local capture controller +now binds metadata and the migration version to the same SQL read view, checks +profile relationships without full ID maps, and captures all thirteen raw tables +into bounded binary frames. SQLite generated cases and a MySQL independent-writer +fixture exercise capture and cleanup. Canonical portable semantic validation, +authenticated client/server integration, larger-wallet resource policy and +measured physical storage costs remain open. The migration and local controller +do not expose or advertise a remote snapshot API. + +The internal directory component now authenticates every page receipt under the +complete root chain and builds a fixed thirteen-table index. SQL reads select +only bounded receipt metadata, retain the original binding bytes and recheck +profile/expiry after I/O. A separate verifier binds expected source/profile/root, +rejects malformed metadata and checks each detached payload against its verified +receipt. Independent SHA-256 fixtures, 300 generated directory cases, maximum +size boundaries and cross-connection SQL capture reads exercise this contract. +This advances S3 table positioning without adding a remote endpoint, negotiating +capabilities or establishing portable semantic/proof validation. + +The durable request-lifecycle foundation reserves shared capacity before source +acquisition and atomically assigns an archive and publishes its ready receipt. +Its immutable deadline is part of request identity, so bounded receipt collection +does not allow an old request to reopen. A separate auxiliary migration retains +bounded terminal history and cleanup ownership. SQLite fault and generated +schedule tests cover these persistence rules. The local service controller now +integrates the existing owned source slot and shared capture path, reserves before +pool acquisition and drains cancellation/shutdown before releasing its admission. +Ready publication follows physical reader cleanup; failed cleanup fences the +controller, and completed archives survive replacement. + +The subsequent authenticated HTTP layer now negotiates the migrated WAL/MySQL +archive capability, returns prompt durable admission and accepts exact profile-bound +status/directory/page/cancel requests. Full and mobile client transports enforce +a dedicated response cap before parsing and bind immutable receipts, source +metadata and page inclusion. Actual synthetic HTTP tests cover lost admission, +replacement, cross-profile refusal and physical shutdown. Admission invokes +expiry cleanup; explicit resource-limit terminal status remains distinct from +other failures. This advances the S3 transport portion. The clock/cursor-compatible +remote row reader, sync-manager adoption, remote destination, portable semantics +and performance qualification under contention remain open. + +The next unadvertised reader implementation adds distinct server-issued v2 +offers, require-existing admission, bounded exact-request retry, fixed client +leases and verified packed row paging. Successful explicit close can collect a +reader receipt without reopening it; failure receipts remain pollable until +cancellation or expiry. The client source adapter and local manager/destination +integration detach, compare and persist optional archive positions and preserve +typed cancellation without treating unrelated failures as cancellation or +fallback. Actual full/mobile HTTP fixtures cover all thirteen row schemas, +replacement, admission loss and cancellation, alongside a generated reader +property. A complete-suite restart failure exposed a reused HTTP connection +reset; immutable requests now permit one native-fetch retry without renewing +the lease or retrying authentication failures. Actual full/mobile HTTP-to-local +sync fixtures retain durable positions on cancellation and complete a later +view into an occupied profile without duplicate labels. The production server reader setting remains absent pending owner +recovery and physical cleanup qualification across replicas. This checkpoint +does not complete S3, V1, remote destination support or portable export/import. + +These checkpoints advance parts of S1/S2/P1/S4. They do not complete primary +reconciliation, indexed identity/update predicates and commit-order high-water +positions, authenticated remote views, durable source views, streaming or staged +restore. Large-row/reference/retention limits still use serialized fallback. +IndexedDB writers wait during capture and legacy archive helpers still materialize +the document. Local MySQL fixtures qualify their isolated version/configuration; +they do not qualify deployed MySQL/PXC behavior or physical mobile execution. + +For every checkpoint record the exact source revision, commands, fixture and +platform, measured result, compatibility result and remaining limitation. Link +evidence to the requirement it actually demonstrates. Native browser evidence +does not qualify physical mobile execution; synthetic signing does not qualify a +funded restore/spend; source CI does not constitute publication or deployment. + +The implementation goal stays active until all required work is complete or an +explicit external dependency prevents further progress. Prepare concrete release, +deployment and funded-drill artifacts before requesting the separate operator +authorization those actions require. PR #569 must remain open even when complete. + +The source-owner fence checkpoint adds an auxiliary exact-claim slot before +service capture and retains archive/request quota through physical cleanup. +Cross-controller cancellation stops the next atomic append; request and direct +archive cleanup both refuse to release outstanding owners. Ready publication +follows the same cleanup acknowledgement. This closes premature logical release +but deliberately does not reclaim unproved process loss: backend-bound recovery +remains required. The client now validates exact pending-cleanup receipts and +uses fixed, bounded cancellation polling while awaiting I/O settlement. The server reader capability stays +off and S3 remains open. + +The backend-guard checkpoint adds eight persistent slot bindings and a versioned +owner migration. New service sources bind local SQLite WAL file identities or +the actual MySQL server/database before their guarded read; old unguarded owners +remain reserved. Recovery takes the same backend guard, fences the exact expired +or terminal claim, physically closes its proof connection and only then releases +ownership. SQLite keeps its guard transaction open through native destruction; +MySQL waits for the socket close event. Focused and generated tests cover delayed +acquisition/close, immutable views, foreground writes, stale acknowledgements, +slot reuse and changed identities. Built-artifact SQLite/MySQL fixtures terminate +an owned synthetic process and recover its claim. This advances S3 orphan +recovery but does not qualify distributed filesystems, PXC, global physical-pool +limits, remote destinations or complete lifecycle/performance behavior. Reader +advertisement remains disabled and the full program remains incomplete. + +Provider lifecycle qualification also preserves an unproved cleanup failure +after the retained read promise settles. The provider keeps its source slot +fenced and destruction exposes that error; an ordinary read failure remains +retryable after proved physical cleanup. Regression tests cover both opening +and closing failures, shared recovery admission, shutdown drainage, immutable +request bindings and exact cancellation outcomes. These checks do not replace +the complete mutation campaign or exact-head hosted qualification. + +The auxiliary profile-index checkpoint adds separate `(table, user, row)` keys +for eight directly owned SQL tables. Source triggers preserve independent-writer +changes and a 256-row bootstrap resumes from atomic key/progress commits. The +standard schema indexes and legacy OFFSET order remain intact. Ordinary readers, +archive pages and closure checks select the complete migration in the same +retained view, preserving all thirteen table values and cursor representations. +Incomplete journaled state refuses new views. This advances indexed profile +selection only: the remaining relationship/global-table work, immutable +commit-order high-water positions, complete tombstone propagation, nonblocking +IndexedDB and the rest of the implementation program remain required. + +The SQLite conflict-maintenance follow-up adds an explicit forward migration for +all four auxiliary index families. Metadata-bound BEFORE/AFTER witnesses preserve +replacement and nested-owner semantics; twelve bounded streams rebuild a fresh +generation under independent writes, and bounded physical-row retirement removes +only obsolete auxiliary tables. Reader generation and journal state are selected +inside the retained view. Historical-migration fixtures remain explicit, while +current ordinary/archive readers and actual registered-migrator process-loss +fixtures exercise adoption and recovery. Explicit full-data deletion remains +supported; normal downgrade refuses without removing standard records. This +advances indexed selection/recovery within S2 and V2. It does not complete +committed-change continuity, large-wallet performance or the remaining program. + +## Owned journal capture checkpoint + +The internal SQL capture controller now owns its writer/reader pools and keeps +provider admission through physical cleanup. It binds the actual backend and +pinned complete generation, profile and schema, commits an exact receipt before +publication, and verifies closure after releasing the writer barrier. WAL and +MySQL native fixtures cover process loss before commit, after durable commit and +after publication. These component proofs advance S2; they do not complete it. +Provider floor/collection integration, runtime quotas, registered +journal migration/recovery, generation-aware delta payload pages and receiver +integration remain required. The other acceptance rows remain open. + +Main's Postgres storage contract is preserved through the legacy path. Snapshot +auxiliary migrations are explicit no-ops there and the associated capabilities +remain unavailable; future support requires new forward migrations. + +The first hosted capture head exposed a dependency-installation mismatch in the +pre-install native-job contract and five analyzer findings. The contract now +checks the exact six-row matrix and pinned Postgres service without a workspace +YAML dependency. Disposable process-cut children receive an exclusive inherited +marker descriptor and cannot choose a filesystem write path through CLI input. +Local source controls and the native capture cuts qualify this remediation; +complete exact-head hosted success is still required before any ready claim. + +## Internal retention floor and collection checkpoint + +Floor advancement now reserves the global writer clock before current retention +and receipt locks and refuses a backwards, future or live-prefix-crossing floor. +Database-clock expiry decides whether a receipt still pins continuity; expired +records stay charged until bounded receipt deletion commits. Collection examines +at most 256 complete primary-key rows, including live/newer entries, validates the +whole page, and deletes only absent metadata at or before the fixed bound floor. +Its cursor binds epoch, floor, stream and exact composite key. + +Native WAL, MySQL RC and RR fixtures prove exact large revisions, active-prefix +pins, current reads through an older transaction snapshot, nonwaiting conflict +refusal, PRIMARY seek plans, retained-view immutability and all thirteen source +tables preserved. Twenty-four real process cuts cover partial deletion and floor +or collection commit with rollback/lost-acknowledgement recovery. Sixty dedicated +tests include 300 independent seeded floor ledgers. The complete journal target +now owns nineteen whole modules in seventeen parts; no campaign threshold, +source/test union or final qualification gate is reduced. + +Provider maintenance integration, runtime quotas and registered forward lifecycle +and recovery remain required within S2. No public reader advertisement or +migration registration is introduced by this checkpoint. Every acceptance row +remains open until its complete implementation and end-to-end evidence exist. + +## Owned bounded journal maintenance checkpoint + +The internal provider controller now owns complete generation validation and +configured migration-owner exclusion before each bounded floor or tombstone-page +transaction. MySQL permits foreground writes during generation validation and +reserves the global clock only for the bounded operation; SQLite obtains the WAL +writer before generation reads. Admission is shared with retained source capture. +Cancellation/result expiry retains that admission through native drain, and +provider destruction fences new work synchronously. Cleanup failures preserve +operation and native causes and stop further retained admission. + +Unit fault tests, 300 seeded lifecycle schedules and native WAL/RC/RR proofs +exercise commit, partial deletion, lost acknowledgement, cancellation, shutdown +and preservation of all thirteen source arrays. The journal mutation registration +now owns twenty-two complete modules in twenty parts, while every part retains +the complete journal test union. The retained-provider test union also includes +the complete maintenance suites. Each inherited snapshot-sync target retains the +complete journal test union for its owned provider ranges. Policies, thresholds +and deadlines are preserved. + +This advances the internal maintenance portion of S2. Runtime quotas, registered +forward generation lifecycle/recovery, payload/receiver integration and all other +#544 acceptance requirements remain open. It adds no migration, advertised +capability or complete-program readiness claim. Raw operator DDL must remain +outside maintenance unless separately excluded by the operator.